man_eb_deployer 0.8.0

data/lib/eb_deployer/default_config.yml

@@ -0,0 +1,159 @@
+# application name
+application: <%= app_name %>
+
+# common settings for all environments
+common:
+  # Solution stack for elastic beanstalk, default is 64bit tomcat 7 for JAVA app
+  # solution_stack_name: 64bit Amazon Linux 2014.03 v1.0.3 running Tomcat 7 Java 7
+
+  # Tier name for environments. Current supported values are WebServer and Worker
+  # tier: WebServer
+
+  # AWS region to deploy. Default to us-east-1
+  # region: us-west-1
+
+  # There are three deployment strategies: 'blue-green', 'blue-only', or 'inplace-update'.
+  # Blue green deployments keep two elastic beanstalk environments and always deploy to
+  # inactive one, to achieve zero downtime.
+  # Blue only deployments do everything that the blue green deployments do except for the final
+  # inactive to active CNAME swap leaving the newly deployed application on the inactive
+  # "blue" instance.
+  # Inplace-update strategy will only keep one environment, and update the version inplace on
+  # deploy. Inplace-update will save resources but will suffer from downtime.
+  # (All old environments need be destroyed when you switching between strategies.)
+  # Default strategy is 'blue-green'.
+  # strategy: blue-green
+
+  # Name of s3 bucket where uploaded application packages will be stored.
+  # Note that the string ".packages" will be added as a suffix to your bucket.
+  # So, if "thoughtworks.simple" is passed as the bucket name, the actual s3 bucket
+  # name will be thoughtworks.simple.packages. Default to application name.
+  # package_bucket: my-s3-bucket
+
+  # If phoenix mode is turned on, it will terminate the old elastic
+  # beanstalk environment and recreate on deploy. For blue-green
+  # deployment it terminates the inactive environment first then
+  # recreate it. This is useful to avoid configuration drift and
+  # accumulating state on the ec2 instances. Default is off but we recommend
+  # it to be turned on for production environment.
+  # phoenix_mode: off
+
+  # The tags you would like to be associated with your resources.
+  # These tags will only be used when you first launch an environment.  If you are using
+  # phoenix_mode set as true each time you deploy you will get a new environment and therefore
+  # any changes to your tags.  If phoenix_mode is false then it will only use your tags on the
+  # initial deploy.
+  # tags:
+  #   my_tag_key: my_tag_value
+
+  # Specifies the maximum number of versions to keep. Older versions are removed
+  # and deleted from the S3 source bucket as well. If specified as zero or not
+  # specified, all versions will be kept.  If a version_prefix is given, only removes
+  # version starting with the prefix.
+  # keep_latest: 200
+
+  # Specifies a prefix to prepend to the version label.
+  # This can be useful if you want to use different binaries for different
+  # environments.
+  # version_prefix:
+
+  # Generating version label for package to be deployed. A readable version label will
+  # provide better traceablity of your deployment process.
+  # By default setting is:
+  # version_label: <%%= package_digest %>
+  # which means using MD5 digest of the package file. If you deploy using build
+  # pipeline tool such as GO, switching to pipline counter is highly suggested to
+  # increase the readability. Following example will read pipeline counter from environment
+  # variable with a fall back to digest for local deployment:
+  # version_label: <%%= ENV['GO_PIPELINE_COUNTER'] || package_digest %>
+
+
+  # Smoke test value should be a piece of ruby code with access to single variable
+  # "host_name" -- environment DNS name. Smoke test snippet will be evaluated at
+  # the end of the deployment for inplace-update deployment. For blue-green
+  # deployment it will run after inactive environment update is completed and before
+  # switching over.
+  # Defining a smoke test is highly recommended for serious usage. By default we use
+  # The simplest one that just be checking server landing page using curl, e.g.
+  smoke_test: |
+    curl_http_code = "curl -s -o /dev/null -w \"%{http_code}\" http://#{host_name}"
+    Timeout.timeout(600) do
+      until ['200', '301', '302'].include?(`#{curl_http_code}`.strip)
+        sleep 5
+      end
+    end
+
+  # List of Elastic Beanstalk health states that will be considered healthy when deploying.
+  # You may want to override default 'Green' with that list, if for example you are using
+  # Elastic Beanstalk worker tier and you don't want eb_deployer to fail if you have messages
+  # in your SQS queue.
+  # By default eb_deployer only considers 'Green' as the healthy state.
+  # For a list of all Elastic Beanstalk health states refer to:
+  # http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features.healthstatus.html#using-features.healthstatus.colors
+  # accepted_healthy_states:
+  #  - Green
+  #  - Yellow
+
+  # Elastic Beanstalk settings that will apply to the environments you are
+  # deploying.
+  # For all available options take a look at
+  # http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options.html
+  option_settings:
+    # Following is an example of set EC2 ssh key name. This allow you ssh into the ec2
+    # instance. See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html
+    # - namespace: aws:autoscaling:launchconfiguration
+    #   option_name: EC2KeyName
+    #   value: <my-ec2-key-name>
+
+    # Following is an example which changes EC2 instance type
+    # from t1.micro (default) to m1.small. Instances with t1.micro type sometime
+    # are not very responsible, so m1.small is suggested for saving time.
+    # But if you care about the marginal cost difference you can comment this out to
+    # go back to t1.micro.
+    - namespace: aws:autoscaling:launchconfiguration
+      option_name: InstanceType
+      value: m1.small
+
+
+  # If resources specified, eb_deployer will use the CloudFormation
+  # template you provide to create a default CloudFormation stack with
+  # name <application_name>-<env-name> for the environment current
+  # deploying. And Outputs of the CloudFormation can be mapped to Elastic Beanstalk
+  # options settings.
+  # keys:
+  #    template => CloudFormation template file with JSON format
+  #    policy => CloudFormation policy file with JSON format
+  #    override_policy => (false) If override_policy is true and a policy file is provided then the policy will temporarily override any existing policy on the resource stack during this update,
+  #                       otherwise the provided policy will replace any existing policy on the resource stack
+  #    inputs => A Hash, input values for the CloudFormation template
+  #    outputs => A Hash with key map to your CloudFormation template outputs and value as elastic beanstalk settings namespace and option_name.
+  #    capabilities => An array. You need set it to ['CAPABILITY_IAM'] if the
+  # template include IAM Instance Profile.
+  resources:
+
+    # For example creating a RDS instance for blue green deployment(check jruby-rails4
+    # sample project which has a working example):
+    # template: config/my_rds.json
+    # inputs:
+    #   DBPassword: <%%= ENV["MYDBPASSWORD"] %>
+    # outputs:
+    #   RDSPassSecurityGroup:
+    #     namespace: aws:autoscaling:launchconfiguration
+    #     option_name: SecurityGroups
+    #   RDSDatabaseConfig:
+    #     namespace: aws:elasticbeanstalk:application:environment
+    #     option_name: databaseConfig
+
+
+# You can define environment here. Each environment can overriden any common settings
+environments:
+  dev:
+    # example for overriding common settings
+    # strategy: inplace-update
+  production:
+    option_settings:
+      # example for overriding common option_settings: providing least redundancy
+      # in production environment.
+      # - namespace: aws:autoscaling:asg
+      #   option_name: MinSize
+      #    value: "2"

data/lib/eb_deployer/deployment_strategy/blue_green.rb

@@ -0,0 +1,79 @@
+module EbDeployer
+  module DeploymentStrategy
+    class BlueGreen
+      def initialize(component)
+        @component = component
+      end
+
+      def test_compatibility(env_create_options)
+        @create_opts = env_create_options
+        tier = env_create_options[:tier]
+
+        if tier && tier.downcase == 'worker'
+          raise "Blue green deployment is not supported for Worker tier"
+        end
+      end
+
+      def deploy(version_label, env_settings, inactive_settings=[])
+
+        if !ebenvs.any?(&method(:active_ebenv?))
+          ebenv('a', @component.cname_prefix).
+            deploy(version_label, env_settings)
+          return
+        end
+
+        active_ebenv = ebenvs.detect(&method(:active_ebenv?))
+        inactive_ebenv = ebenvs.reject(&method(:active_ebenv?)).first
+
+        inactive_ebenv.deploy(version_label, env_settings)
+        active_ebenv.swap_cname_with(inactive_ebenv)
+
+        blue_green_terminate_inactive       = @create_opts[:blue_green_terminate_inactive]
+        blue_green_terminate_inactive_wait  = @create_opts[:blue_green_terminate_inactive_wait]
+        blue_green_terminate_inactive_sleep = @create_opts[:blue_green_terminate_inactive_sleep]
+
+        if blue_green_terminate_inactive
+          active_ebenv.log("Waiting #{blue_green_terminate_inactive_wait}s before terminating environment...")
+
+          # Loop until timeout reached or environment becomes Red
+          count = 0
+          loop do
+            break if count >= blue_green_terminate_inactive_wait or inactive_ebenv.health_state != 'Green'
+            sleep blue_green_terminate_inactive_sleep
+            count += blue_green_terminate_inactive_sleep
+          end
+
+          if inactive_ebenv.health_state == 'Green'
+            active_ebenv.log("Active environment healthy, terminating inactive (black) environment")
+            active_ebenv.terminate
+          else
+            active_ebenv.log("Active environment changed state to unhealthy. Existing (black) environment will not be terminated")
+          end
+
+        end
+
+        unless inactive_settings.empty? || blue_green_terminate_inactive
+          active_ebenv.log("applying inactive settings...")
+          active_ebenv.apply_settings(inactive_settings)
+        end
+      end
+
+      private
+      def active_ebenv?(ebenv)
+        ebenv.cname_prefix == @component.cname_prefix
+      end
+
+      def ebenvs
+        [ebenv('a'), ebenv('b')]
+      end
+
+      def ebenv(suffix, cname_prefix=nil)
+        @component.new_eb_env(suffix, cname_prefix || inactive_cname_prefix)
+      end
+
+      def inactive_cname_prefix
+        "#{@component.cname_prefix}-inactive"
+      end
+    end
+  end
+end

data/lib/eb_deployer/deployment_strategy/blue_only.rb

@@ -0,0 +1,45 @@
+module EbDeployer
+  module DeploymentStrategy
+    class BlueOnly
+      def initialize(component)
+        @component = component
+      end
+
+      def test_compatibility(env_create_options)
+        tier = env_create_options[:tier]
+        if tier && tier.downcase == 'worker'
+          raise "Blue only deployment is not supported for Worker tier"
+        end
+      end
+
+      def deploy(version_label, env_settings, inactive_settings=[])
+        if !ebenvs.any?(&method(:active_ebenv?))
+          ebenv('a', @component.cname_prefix).
+            deploy(version_label, env_settings)
+          return
+        end
+
+        inactive_ebenv = ebenvs.reject(&method(:active_ebenv?)).first
+
+        inactive_ebenv.deploy(version_label, env_settings)
+      end
+
+      private
+      def active_ebenv?(ebenv)
+        ebenv.cname_prefix == @component.cname_prefix
+      end
+
+      def ebenvs
+        [ebenv('a'), ebenv('b')]
+      end
+
+      def ebenv(suffix, cname_prefix=nil)
+        @component.new_eb_env(suffix, cname_prefix || inactive_cname_prefix)
+      end
+
+      def inactive_cname_prefix
+        "#{@component.cname_prefix}-inactive"
+      end
+    end
+  end
+end

data/lib/eb_deployer/deployment_strategy/inplace_update.rb

@@ -0,0 +1,16 @@
+module EbDeployer
+  module DeploymentStrategy
+    class InplaceUpdate
+      def initialize(component)
+        @component = component
+      end
+
+      def test_compatibility(env_create_opts)
+      end
+
+      def deploy(version_label, env_settings, inactive_settings)
+        @component.new_eb_env.deploy(version_label, env_settings)
+      end
+    end
+  end
+end

data/lib/eb_deployer/deployment_strategy.rb

@@ -0,0 +1,20 @@
+require 'eb_deployer/deployment_strategy/inplace_update'
+require 'eb_deployer/deployment_strategy/blue_green'
+require 'eb_deployer/deployment_strategy/blue_only'
+
+module EbDeployer
+  module DeploymentStrategy
+    def self.create(component, strategy_name)
+      case strategy_name.to_s
+      when 'inplace_update', 'inplace-update'
+        InplaceUpdate.new(component)
+      when 'blue_green', 'blue-green'
+        BlueGreen.new(component)
+      when 'blue_only', 'blue-only'
+        BlueOnly.new(component)
+      else
+        raise 'strategy_name: ' + strategy_name.to_s + ' not supported'
+      end
+    end
+  end
+end

data/lib/eb_deployer/eb_environment.rb

@@ -0,0 +1,204 @@
+module EbDeployer
+  class EbEnvironment
+    include Utils
+
+    attr_reader :app, :name
+    attr_writer :event_poller
+
+    def self.unique_ebenv_name(env_name, app_name)
+      raise "Environment name #{env_name} is too long, it must be under 32 chars" if env_name.size > 32
+      digest = Digest::SHA1.hexdigest(app_name + '-' + env_name)[0..6]
+      "#{env_name}-#{digest}"
+    end
+
+    def initialize(app, name, eb_driver, creation_opts={})
+      @app = app
+      @name = self.class.unique_ebenv_name(name, app)
+      @bs = eb_driver
+      @creation_opts = default_create_options.merge(reject_nil(creation_opts))
+      @accepted_healthy_states = @creation_opts[:accepted_healthy_states]
+      @event_poller = nil
+    end
+
+    def deploy(version_label, settings={})
+      terminate if @creation_opts[:phoenix_mode]
+
+      if @bs.environment_exists?(@app, @name)
+        update_eb_env(settings, version_label)
+      else
+        create_eb_env(settings, version_label)
+      end
+
+      smoke_test
+      wait_for_env_become_healthy
+    end
+
+    def apply_settings(settings)
+      raise "Env #{self.name} not exists for applying settings" unless @bs.environment_exists?(@app, @name)
+      wait_for_env_status_to_be_ready
+      with_polling_events(/Successfully deployed new configuration to environment/i) do
+        @bs.update_environment_settings(@app, @name, settings)
+      end
+    end
+
+    def cname_prefix
+      @bs.environment_cname_prefix(@app, @name)
+    end
+
+    def swap_cname_with(another)
+      log("Swap CNAME with env #{another.name}")
+      with_polling_events(/Completed swapping CNAMEs for environments/i) do
+        @bs.environment_swap_cname(self.app, self.name, another.name)
+      end
+    end
+
+    def log(msg)
+      puts "[#{Time.now.utc}][environment:#{@name}] #{msg}"
+    end
+
+    def terminate
+      if @bs.environment_exists?(@app, @name)
+        with_polling_events(/terminateEnvironment completed successfully/i) do
+          @bs.delete_environment(@app, @name)
+        end
+      end
+    end
+
+    def health_state
+      @bs.environment_health_state(@app, @name)
+    end
+
+    private
+
+    def configured_tier
+      @creation_opts[:tier]
+    end
+
+    def template_name
+      @creation_opts[:template_name]
+    end
+
+    def has_cname?
+      !configured_tier || configured_tier.downcase == 'webserver'
+    end
+
+    def configured_cname_prefix
+      @creation_opts[:cname_prefix]
+    end
+
+    def create_eb_env(settings, version_label)
+      solution_stack = @creation_opts[:solution_stack]
+      tags = convert_tags_hash_to_array(@creation_opts.delete(:tags))
+      validate_solutions_stack(solution_stack)
+      with_polling_events(/Successfully launched environment/i) do
+        @bs.create_environment(@app,
+                               @name,
+                               solution_stack,
+                               has_cname? ? configured_cname_prefix : nil,
+                               version_label,
+                               configured_tier,
+                               tags,
+                               settings,
+                               template_name)
+      end
+    end
+
+    def update_eb_env(settings, version_label)
+      with_polling_events(/Successfully deployed new configuration to environment/i) do
+        @bs.update_environment(@app,
+                               @name,
+                               version_label,
+                               configured_tier,
+                               settings,
+                               template_name)
+      end
+    end
+
+    def validate_solutions_stack(stack_name)
+      names = @bs.list_solution_stack_names
+      raise "'#{stack_name}' is not a valid solution stack name, available solution stack names are: #{names.join(', ')}" unless names.include?(stack_name)
+    end
+
+    def smoke_test
+      host_name = @bs.environment_cname(@app, @name)
+      SmokeTest.new(@creation_opts[:smoke_test]).run(host_name, self)
+    end
+
+    def with_polling_events(terminate_pattern, &block)
+      anchor = event_poller.get_anchor
+      yield
+      event_poller.poll(anchor) do |event|
+        if event[:message] =~ /Failed to deploy application/
+          raise event[:message]
+        end
+
+        if event[:message] =~ /Command failed on instance/
+          raise "Elasticbeanstalk instance provision failed (maybe a problem with your .ebextension files). The original message: #{event[:message]}"
+        end
+
+        if event[:message] =~ /complete, but with errors/
+          raise event[:message]
+        end
+
+        if event[:message] =~ /However, there were issues during launch\. See event log for details\./
+          raise "Environment launched, but with errors.  The original message: #{event[:message]}"
+        end
+
+        log_event(event)
+        break if event[:message] =~ terminate_pattern
+      end
+    end
+
+    def convert_tags_hash_to_array tags
+      tags ||= {}
+      tags.inject([]) do |arr, (k, v)|
+        arr << {:key => k, :value => v}
+        arr
+      end
+    end
+
+    def wait_for_env_status_to_be_ready
+      Timeout.timeout(600) do
+        current_status = @bs.environment_status(@app, @name)
+
+        while current_status.downcase != 'ready'
+          log("Environment status: #{current_status}")
+          sleep 15
+          current_status = @bs.environment_status(@app, @name)
+        end
+
+        log("Environment status: #{current_status}")
+      end
+    end
+
+    def wait_for_env_become_healthy
+      Timeout.timeout(600) do
+        current_health_status = @bs.environment_health_state(@app, @name)
+        while !@accepted_healthy_states.include?(current_health_status)
+          log("health status: #{current_health_status}")
+          sleep 15
+          current_health_status = @bs.environment_health_state(@app, @name)
+        end
+
+        log("health status: #{current_health_status}")
+      end
+    end
+
+    def event_poller
+      @event_poller || EventPoller.new(EbEventSource.new(@app, @name, @bs))
+    end
+
+    def default_create_options
+      {
+        :solution_stack => "64bit Amazon Linux 2014.09 v1.1.0 running Tomcat 7 Java 7",
+        :smoke_test =>  Proc.new {},
+        :tier => 'WebServer',
+        :accepted_healthy_states => ['Green']
+      }
+    end
+
+    def log_event(event)
+      puts "[#{event[:event_date]}][environment:#{@name}] #{event[:message]}"
+    end
+  end
+end

data/lib/eb_deployer/eb_event_source.rb

@@ -0,0 +1,35 @@
+module EbDeployer
+  class EbEventSource
+    def initialize(app, env, eb_driver)
+      @app, @env, @eb_driver = app, env, eb_driver
+    end
+
+    def get_anchor
+      events, _ = fetch_events_from_eb(:max_records => 1)
+      events.first
+    end
+
+    def fetch_events(from_anchor, &block)
+      options = {}
+      if from_anchor && from_anchor[:event_date]
+        options[:start_time] = from_anchor[:event_date].iso8601
+      end
+      events, next_token = fetch_events_from_eb(options)
+      should_continue = yield(events)
+      fetch_next(next_token, &block) if next_token && should_continue
+    end
+
+    private
+
+    def fetch_next(next_token, &block)
+      events, next_token = fetch_events_from_eb(:next_token => next_token)
+      should_continue = yield(events)
+      fetch_next(next_token, &block) if next_token && should_continue
+    end
+
+    def fetch_events_from_eb(options)
+      @eb_driver.fetch_events(@app, @env, options)
+    end
+
+  end
+end

data/lib/eb_deployer/environment.rb

@@ -0,0 +1,60 @@
+module EbDeployer
+  class Environment
+    include Utils
+    attr_accessor :creation_opts, :strategy_name
+
+    attr_writer :resource_stacks, :settings, :inactive_settings, :component_under_deploy
+
+    attr_reader :name
+
+    def initialize(app, name, stack_name, eb_driver, &block)
+      @app = app
+      @name = name
+      @stack_name = stack_name
+      @eb_driver = eb_driver
+      @creation_opts = {}
+      @settings = []
+      @inactive_settings = []
+      @strategy_name = :blue_green
+      @components = nil
+      yield(self) if block_given?
+      unless @components
+        @components = [DefaultComponent.new(self, @creation_opts, @strategy_name, @eb_driver)]
+      end
+    end
+
+    def app_name
+      @app.name
+    end
+
+    def deploy(version_label)
+      resource_settings = @resource_stacks.provision(@stack_name)
+      components_to_deploy.each do |component|
+        component.deploy(version_label, @settings + resource_settings, @inactive_settings)
+      end
+    end
+
+    def components=(components_attrs)
+      return unless components_attrs
+      @components = components_attrs.map do |attrs|
+        attrs = symbolize_keys(attrs)
+        Component.new(attrs.delete(:name), self, attrs, @eb_driver)
+      end
+    end
+
+    private
+    def components_to_deploy
+      if @component_under_deploy
+        component = component_named(@component_under_deploy)
+        raise "'#{@component_under_deploy}' is not in the configuration. Available components are #{@components.map(&:name) }" unless component
+        [component]
+      else
+        @components
+      end
+    end
+
+    def component_named(name)
+      @components.detect { |c| c.name == name }
+    end
+  end
+end

data/lib/eb_deployer/event_poller.rb

@@ -0,0 +1,51 @@
+module EbDeployer
+  class EventPoller
+    include Utils
+    POLL_INTERVAL = 15
+
+    def initialize(event_source)
+      @event_source = event_source
+    end
+
+    def get_anchor
+      @event_source.get_anchor
+    end
+
+    def poll(from_anchor, &block)
+      handled = Set.new
+      loop do
+        @event_source.fetch_events(from_anchor) do |events|
+          # events from event source is latest first order
+          to_be_handled = []
+          reached_anchor = false
+
+          events.each do |event|
+            if digest(event) == digest(from_anchor)
+              reached_anchor = true
+            end
+
+            if !handled.include?(digest(event)) && !reached_anchor
+              to_be_handled << event
+            end
+          end
+
+          to_be_handled.reverse.each do |event|
+            yield(event)
+            handled << digest(event)
+          end
+
+          !reached_anchor
+        end
+        sleep POLL_INTERVAL
+      end
+    end
+
+    private
+
+    def digest(event)
+      return nil unless event
+      event = event.to_h if event.respond_to?(:to_h)
+      event.to_json
+    end
+  end
+end