eb_deployer_updated 0.8.0

data/lib/eb_deployer/aws_driver/cloud_formation_driver.rb

@@ -0,0 +1,53 @@
+module EbDeployer
+  module AWSDriver
+    class CloudFormationDriver
+
+      def initialize
+        @client = Aws::CloudFormation::Client.new
+      end
+
+      def stack_exists?(name)
+        describe_stack(name)
+        true
+      rescue Aws::CloudFormation::Errors::ValidationError
+        false
+      end
+
+      def create_stack(name, template, opts)
+        @client.create_stack(opts.merge(:stack_name => name,
+                                        :template_body => template,
+                                        :parameters => convert_parameters(opts[:parameters])))
+      end
+
+      def update_stack(name, template, opts)
+        @client.update_stack(opts.merge(:stack_name => name,
+                                        :template_body => template,
+                                        :parameters => convert_parameters(opts[:parameters])))
+      end
+
+      def query_output(name, key)
+        output = describe_stack(name)[:outputs].find { |o| o[:output_key] == key }
+        output && output[:output_value]
+      end
+
+      def fetch_events(name, options={})
+        response = @client.describe_stack_events(options.merge(:stack_name => name))
+        return response.stack_events, response.next_token
+      end
+
+      private
+
+      def describe_stack(name)
+        @client.describe_stacks(:stack_name => name)[:stacks].first
+      end
+
+      def log(msg)
+        puts "[#{Time.now.utc}][cloud_formation_driver] #{msg}"
+      end
+
+      def convert_parameters(params)
+        params.map { |k, v| {:parameter_key => k, :parameter_value => v}}
+      end
+    end
+  end
+end

data/lib/eb_deployer/aws_driver/s3_driver.rb

@@ -0,0 +1,35 @@
+module EbDeployer
+  module AWSDriver
+    class S3Driver
+      def create_bucket(bucket_name)
+        s3.create_bucket(:bucket => bucket_name)
+      end
+
+      def bucket_exists?(bucket_name)
+        s3.bucket(bucket_name).exists?
+      end
+
+      def object_length(bucket_name, obj_name)
+        obj(bucket_name, obj_name).content_length rescue nil
+      end
+
+      def upload_file(bucket_name, obj_name, file)
+        o = obj(bucket_name, obj_name)
+        o.upload_file(file)
+      end
+
+      private
+      def s3
+        Aws::S3::Resource.new(client: Aws::S3::Client.new)
+      end
+
+      def obj(bucket_name, obj_name)
+        s3.bucket(bucket_name).object(obj_name)
+      end
+
+      def buckets
+        s3.buckets
+      end
+    end
+  end
+end

data/lib/eb_deployer/aws_driver.rb

@@ -0,0 +1,8 @@
+require 'eb_deployer/aws_driver/s3_driver'
+require 'eb_deployer/aws_driver/beanstalk'
+require 'eb_deployer/aws_driver/cloud_formation_driver'
+
+module EbDeployer
+  module AWSDriver
+  end
+end

data/lib/eb_deployer/cf_event_source.rb

@@ -0,0 +1,26 @@
+module EbDeployer
+  class CfEventSource
+    def initialize(stack_name, cf_driver)
+      @stack_name = stack_name
+      @cf_driver = cf_driver
+    end
+
+    def get_anchor
+      events, _ = @cf_driver.fetch_events(@stack_name)
+      events.first
+    end
+
+    def fetch_events(from_anchor, &block)
+      events, next_token = @cf_driver.fetch_events(@stack_name)
+      should_continue = yield(events)
+      fetch_next(next_token, &block) if next_token && should_continue
+    end
+
+    private
+    def fetch_next(next_token, &block)
+      events, next_token = @cf_driver.fetch_events(@stack_name, :next_token => next_token)
+      should_continue = yield(events)
+      fetch_next(next_token, &block) if next_token && should_continue
+    end
+  end
+end

data/lib/eb_deployer/cloud_formation_provisioner.rb

@@ -0,0 +1,120 @@
+module EbDeployer
+  class ResourceNotInReadyState < StandardError
+  end
+
+  class CloudFormationProvisioner
+    SUCCESS_STATS = ["CREATE_COMPLETE", "UPDATE_COMPLETE"]
+    FAILED_STATS = ["CREATE_FAILED", "UPDATE_FAILED", "UPDATE_ROLLBACK_COMPLETE"]
+
+    def initialize(stack_name, cf_driver)
+      @stack_name = stack_name
+      @cf_driver = cf_driver
+      @poller = EventPoller.new(CfEventSource.new(@stack_name, @cf_driver))
+    end
+
+    def provision(resources, tags)
+      resources = symbolize_keys(resources)
+      template = File.read(resources[:template])
+      capabilities = resources[:capabilities] || []
+      params = resources[:inputs] || resources[:parameters] || {}
+      policy = File.read(resources[:policy]) if resources[:policy]
+      override_policy = resources[:override_policy] || false
+      anchor = nil
+      begin
+        if stack_exists?
+          anchor = @poller.get_anchor
+          update_stack(template, params, capabilities, policy, override_policy, tags)
+        else
+          create_stack(template, params, capabilities, policy, tags)
+        end
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        if e.message =~ /No updates are to be performed/
+          log(e.message)
+          return
+        else
+          raise
+        end
+      end
+      wait_for_stack_op_terminate(anchor)
+      log("Resource stack provisioned successfully")
+    end
+
+    def transform_outputs(resources)
+      resources = symbolize_keys(resources)
+      outputs = resources[:outputs] || {}
+      transforms = resources[:transforms] || {}
+      transform_output_to_settings(convert_to_transforms(outputs).merge(transforms))
+    end
+
+    def output(key)
+      @cf_driver.query_output(@stack_name, key)
+    rescue Aws::CloudFormation::Errors::ValidationError
+      raise ResourceNotInReadyState.new("Resource stack not in ready state yet, perhaps you should provision it first?")
+    end
+
+    private
+
+    #todo: remove duplication
+    def symbolize_keys(hash)
+      hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+    end
+
+
+    def convert_to_transforms(outputs)
+      outputs.inject({}) do |memo, (key, value)|
+        memo[key] = lambda { |output_value| value.merge('value' => output_value) }
+        memo
+      end
+    end
+
+    def update_stack(template, params, capabilities, policy, override_policy, tags)
+      opts = {:capabilities => capabilities, :parameters => params, :tags => tags}
+      if (policy)
+        opts[:stack_policy_during_update_body] = policy if override_policy
+        log("Using temporary stack policy to apply resource stack updates") if override_policy
+        opts[:stack_policy_body] = policy unless override_policy
+        log("Applying new stack policy to existing resource stack") unless override_policy
+      end
+      @cf_driver.update_stack(@stack_name, template, opts)
+    end
+
+    def stack_exists?
+      @cf_driver.stack_exists?(@stack_name)
+    end
+
+    def create_stack(template, params, capabilities, policy, tags)
+      opts = {:disable_rollback => true, :capabilities => capabilities, :parameters => params, :tags => tags}
+      opts[:stack_policy_body] = policy if policy
+      log("Applying stack policy to new resource stack") if policy
+      @cf_driver.create_stack(@stack_name, template, opts)
+    end
+
+    def transform_output_to_settings(transforms)
+      (transforms || []).inject([]) do |settings, pair|
+        key, transform = pair
+        settings << transform.call(output(key))
+        settings
+      end.flatten
+    end
+
+    def wait_for_stack_op_terminate(anchor)
+      @poller.poll(anchor) do |event|
+        log_event(event)
+        if FAILED_STATS.include?(event.resource_status)
+          raise "Resource stack update failed!"
+        end
+
+        break if event.logical_resource_id == @stack_name && SUCCESS_STATS.include?(event.resource_status)
+      end
+    end
+
+    def log_event(event)
+      puts "[#{event.timestamp}][cloud_formation_provisioner] #{event.resource_type}(#{event.logical_resource_id}) #{event.resource_status} \"#{event.resource_status_reason}\""
+    end
+
+
+    def log(msg)
+      puts "[#{Time.now.utc}][cloud_formation_provisioner] #{msg}"
+    end
+  end
+end

data/lib/eb_deployer/component.rb

@@ -0,0 +1,45 @@
+module EbDeployer
+  class Component
+    attr_reader :name
+
+    def initialize(name, env, options, eb_driver)
+      @name = name
+      @env = env
+      @eb_driver = eb_driver
+      @options = options.dup
+      @component_eb_settings = @options.delete(:option_settings) || []
+      @component_inactive_settings = @options.delete(:inactive_settings) || []
+      strategy_name = @options[:strategy] || @env.strategy_name
+      @strategy = DeploymentStrategy.create(self, strategy_name)
+    end
+
+    def cname_prefix
+      @options[:cname_prefix] || default_cname_prefix
+    end
+
+    def deploy(version_label, eb_settings, inactive_settings=[])
+      @strategy.test_compatibility(create_options)
+      @strategy.deploy(version_label,
+                       eb_settings + @component_eb_settings,
+                       inactive_settings + @component_inactive_settings)
+    end
+
+    def new_eb_env(suffix=nil, cname_prefix_overriding=nil)
+      EbEnvironment.new(@env.app_name,
+                        [@env.name, @name, suffix].compact.join('-'),
+                        @eb_driver,
+                        create_options.merge(:cname_prefix => cname_prefix_overriding || cname_prefix))
+
+    end
+
+    private
+
+    def create_options
+      @env.creation_opts.merge(@options)
+    end
+
+    def default_cname_prefix
+      [@env.app_name, @env.name, @name].join('-')
+    end
+  end
+end

data/lib/eb_deployer/config_loader.rb

@@ -0,0 +1,64 @@
+require 'securerandom'
+require 'digest'
+require 'yaml'
+
+module EbDeployer
+  class ConfigLoader
+    include Utils
+
+    class EvalBinding
+      attr_reader :environment, :package_digest
+      def initialize(package_digest, env)
+        @package_digest = package_digest
+        @environment = env
+      end
+
+      def random_hash
+        SecureRandom.hex[0..9]
+      end
+    end
+
+    def load(options)
+      options = options.dup
+      package_digest = package_digest(options[:package])
+      config_file = options.delete(:config_file)
+
+      env = options[:environment]
+      config_settings = load_config_settings(config_file, package_digest, env)
+
+      app_name = config_settings[:application]
+
+      common_settings = symbolize_keys(config_settings[:common])
+      common_settings[:version_label] ||= package_digest
+
+      envs = config_settings[:environments]
+      raise "Environment #{env} is not defined in #{config_file}" unless envs.has_key?(env)
+      env_settings = symbolize_keys(envs[env] || {})
+      env_option_settings = env_settings.delete(:option_settings) || []
+
+      ret = options.merge(common_settings).merge(env_settings)
+      ret[:application] = app_name
+      ret[:option_settings] ||= []
+      ret[:option_settings] += env_option_settings
+      ret
+    end
+
+    private
+
+    def load_config_settings(config_file, package_digest, env)
+      yaml = ERB.new(File.read(config_file)).
+             result(eval_binding(package_digest, env))
+      symbolize_keys(YAML.load(yaml))
+    end
+
+    def eval_binding(package_digest, env)
+      EvalBinding.new(package_digest, env).instance_eval { binding }
+    end
+
+    def package_digest(package)
+      return nil unless package
+      return package unless File.exist?(package)
+      Digest::MD5.file(package).hexdigest
+    end
+  end
+end

data/lib/eb_deployer/default_component.rb

@@ -0,0 +1,32 @@
+module EbDeployer
+  class DefaultComponent
+    def initialize(env, creation_opts, strategy_name, eb_driver)
+      @env = env
+      @eb_driver = eb_driver
+      @creation_opts = creation_opts
+      @strategy = DeploymentStrategy.create(self, strategy_name)
+    end
+
+    def cname_prefix
+      @creation_opts[:cname_prefix] || default_cname_prefix
+    end
+
+    def deploy(version_label, eb_settings, inactive_settings=[])
+      @strategy.test_compatibility(@creation_opts)
+      @strategy.deploy(version_label, eb_settings, inactive_settings)
+    end
+
+    def new_eb_env(suffix=nil, cname_prefix_overriding=nil)
+      EbEnvironment.new(@env.app_name,
+                        [@env.name, suffix].compact.join('-'),
+                        @eb_driver,
+                        @creation_opts.merge(:cname_prefix => cname_prefix_overriding || cname_prefix))
+    end
+
+    private
+
+    def default_cname_prefix
+      [@env.app_name, @env.name].join('-')
+    end
+  end
+end

data/lib/eb_deployer/default_config.rb

@@ -0,0 +1,20 @@
+module EbDeployer
+  class DefaultConfig
+    attr_reader :app_name
+
+    def initialize(app_name)
+      @app_name = app_name.gsub('_', '-')
+    end
+
+    def write_to(path)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.open(path, 'w') { |f| f << ERB.new(File.read(config_template)).result(binding) }
+    end
+
+    private
+
+    def config_template
+      File.expand_path("../default_config.yml", __FILE__)
+    end
+  end
+end

data/lib/eb_deployer/default_config.yml

@@ -0,0 +1,159 @@
+# application name
+application: <%= app_name %>
+
+# common settings for all environments
+common:
+  # Solution stack for elastic beanstalk, default is 64bit tomcat 7 for JAVA app
+  # solution_stack_name: 64bit Amazon Linux 2014.03 v1.0.3 running Tomcat 7 Java 7
+
+  # Tier name for environments. Current supported values are WebServer and Worker
+  # tier: WebServer
+
+  # AWS region to deploy. Default to us-east-1
+  # region: us-west-1
+
+  # There are three deployment strategies: 'blue-green', 'blue-only', or 'inplace-update'.
+  # Blue green deployments keep two elastic beanstalk environments and always deploy to
+  # inactive one, to achieve zero downtime.
+  # Blue only deployments do everything that the blue green deployments do except for the final
+  # inactive to active CNAME swap leaving the newly deployed application on the inactive
+  # "blue" instance.
+  # Inplace-update strategy will only keep one environment, and update the version inplace on
+  # deploy. Inplace-update will save resources but will suffer from downtime.
+  # (All old environments need be destroyed when you switching between strategies.)
+  # Default strategy is 'blue-green'.
+  # strategy: blue-green
+
+  # Name of s3 bucket where uploaded application packages will be stored.
+  # Note that the string ".packages" will be added as a suffix to your bucket.
+  # So, if "thoughtworks.simple" is passed as the bucket name, the actual s3 bucket
+  # name will be thoughtworks.simple.packages. Default to application name.
+  # package_bucket: my-s3-bucket
+
+  # If phoenix mode is turned on, it will terminate the old elastic
+  # beanstalk environment and recreate on deploy. For blue-green
+  # deployment it terminates the inactive environment first then
+  # recreate it. This is useful to avoid configuration drift and
+  # accumulating state on the ec2 instances. Default is off but we recommend
+  # it to be turned on for production environment.
+  # phoenix_mode: off
+
+  # The tags you would like to be associated with your resources.
+  # These tags will only be used when you first launch an environment.  If you are using
+  # phoenix_mode set as true each time you deploy you will get a new environment and therefore
+  # any changes to your tags.  If phoenix_mode is false then it will only use your tags on the
+  # initial deploy.
+  # tags:
+  #   my_tag_key: my_tag_value
+
+  # Specifies the maximum number of versions to keep. Older versions are removed
+  # and deleted from the S3 source bucket as well. If specified as zero or not
+  # specified, all versions will be kept.  If a version_prefix is given, only removes
+  # version starting with the prefix.
+  # keep_latest: 200
+
+  # Specifies a prefix to prepend to the version label.
+  # This can be useful if you want to use different binaries for different
+  # environments.
+  # version_prefix:
+
+  # Generating version label for package to be deployed. A readable version label will
+  # provide better traceablity of your deployment process.
+  # By default setting is:
+  # version_label: <%%= package_digest %>
+  # which means using MD5 digest of the package file. If you deploy using build
+  # pipeline tool such as GO, switching to pipline counter is highly suggested to
+  # increase the readability. Following example will read pipeline counter from environment
+  # variable with a fall back to digest for local deployment:
+  # version_label: <%%= ENV['GO_PIPELINE_COUNTER'] || package_digest %>
+
+
+  # Smoke test value should be a piece of ruby code with access to single variable
+  # "host_name" -- environment DNS name. Smoke test snippet will be evaluated at
+  # the end of the deployment for inplace-update deployment. For blue-green
+  # deployment it will run after inactive environment update is completed and before
+  # switching over.
+  # Defining a smoke test is highly recommended for serious usage. By default we use
+  # The simplest one that just be checking server landing page using curl, e.g.
+  smoke_test: |
+    curl_http_code = "curl -s -o /dev/null -w \"%{http_code}\" http://#{host_name}"
+    Timeout.timeout(600) do
+      until ['200', '301', '302'].include?(`#{curl_http_code}`.strip)
+        sleep 5
+      end
+    end
+
+  # List of Elastic Beanstalk health states that will be considered healthy when deploying.
+  # You may want to override default 'Green' with that list, if for example you are using
+  # Elastic Beanstalk worker tier and you don't want eb_deployer to fail if you have messages
+  # in your SQS queue.
+  # By default eb_deployer only considers 'Green' as the healthy state.
+  # For a list of all Elastic Beanstalk health states refer to:
+  # http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features.healthstatus.html#using-features.healthstatus.colors
+  # accepted_healthy_states:
+  #  - Green
+  #  - Yellow
+
+  # Elastic Beanstalk settings that will apply to the environments you are
+  # deploying.
+  # For all available options take a look at
+  # http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options.html
+  option_settings:
+    # Following is an example of set EC2 ssh key name. This allow you ssh into the ec2
+    # instance. See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html
+    # - namespace: aws:autoscaling:launchconfiguration
+    #   option_name: EC2KeyName
+    #   value: <my-ec2-key-name>
+
+    # Following is an example which changes EC2 instance type
+    # from t1.micro (default) to m1.small. Instances with t1.micro type sometime
+    # are not very responsible, so m1.small is suggested for saving time.
+    # But if you care about the marginal cost difference you can comment this out to
+    # go back to t1.micro.
+    - namespace: aws:autoscaling:launchconfiguration
+      option_name: InstanceType
+      value: m1.small
+
+
+  # If resources specified, eb_deployer will use the CloudFormation
+  # template you provide to create a default CloudFormation stack with
+  # name <application_name>-<env-name> for the environment current
+  # deploying. And Outputs of the CloudFormation can be mapped to Elastic Beanstalk
+  # options settings.
+  # keys:
+  #    template => CloudFormation template file with JSON format
+  #    policy => CloudFormation policy file with JSON format
+  #    override_policy => (false) If override_policy is true and a policy file is provided then the policy will temporarily override any existing policy on the resource stack during this update,
+  #                       otherwise the provided policy will replace any existing policy on the resource stack
+  #    inputs => A Hash, input values for the CloudFormation template
+  #    outputs => A Hash with key map to your CloudFormation template outputs and value as elastic beanstalk settings namespace and option_name.
+  #    capabilities => An array. You need set it to ['CAPABILITY_IAM'] if the
+  # template include IAM Instance Profile.
+  resources:
+
+    # For example creating a RDS instance for blue green deployment(check jruby-rails4
+    # sample project which has a working example):
+    # template: config/my_rds.json
+    # inputs:
+    #   DBPassword: <%%= ENV["MYDBPASSWORD"] %>
+    # outputs:
+    #   RDSPassSecurityGroup:
+    #     namespace: aws:autoscaling:launchconfiguration
+    #     option_name: SecurityGroups
+    #   RDSDatabaseConfig:
+    #     namespace: aws:elasticbeanstalk:application:environment
+    #     option_name: databaseConfig
+
+
+# You can define environment here. Each environment can overriden any common settings
+environments:
+  dev:
+    # example for overriding common settings
+    # strategy: inplace-update
+  production:
+    option_settings:
+      # example for overriding common option_settings: providing least redundancy
+      # in production environment.
+      # - namespace: aws:autoscaling:asg
+      #   option_name: MinSize
+      #    value: "2"

data/lib/eb_deployer/deployment_strategy/blue_green.rb

@@ -0,0 +1,79 @@
+module EbDeployer
+  module DeploymentStrategy
+    class BlueGreen
+      def initialize(component)
+        @component = component
+      end
+
+      def test_compatibility(env_create_options)
+        @create_opts = env_create_options
+        tier = env_create_options[:tier]
+
+        if tier && tier.downcase == 'worker'
+          raise "Blue green deployment is not supported for Worker tier"
+        end
+      end
+
+      def deploy(version_label, env_settings, inactive_settings=[])
+
+        if !ebenvs.any?(&method(:active_ebenv?))
+          ebenv('a', @component.cname_prefix).
+            deploy(version_label, env_settings)
+          return
+        end
+
+        active_ebenv = ebenvs.detect(&method(:active_ebenv?))
+        inactive_ebenv = ebenvs.reject(&method(:active_ebenv?)).first
+
+        inactive_ebenv.deploy(version_label, env_settings)
+        active_ebenv.swap_cname_with(inactive_ebenv)
+
+        blue_green_terminate_inactive       = @create_opts[:blue_green_terminate_inactive]
+        blue_green_terminate_inactive_wait  = @create_opts[:blue_green_terminate_inactive_wait]
+        blue_green_terminate_inactive_sleep = @create_opts[:blue_green_terminate_inactive_sleep]
+
+        if blue_green_terminate_inactive
+          active_ebenv.log("Waiting #{blue_green_terminate_inactive_wait}s before terminating environment...")
+
+          # Loop until timeout reached or environment becomes Red
+          count = 0
+          loop do
+            break if count >= blue_green_terminate_inactive_wait or inactive_ebenv.health_state != 'Green'
+            sleep blue_green_terminate_inactive_sleep
+            count += blue_green_terminate_inactive_sleep
+          end
+
+          if inactive_ebenv.health_state == 'Green'
+            active_ebenv.log("Active environment healthy, terminating inactive (black) environment")
+            active_ebenv.terminate
+          else
+            active_ebenv.log("Active environment changed state to unhealthy. Existing (black) environment will not be terminated")
+          end
+
+        end
+
+        unless inactive_settings.empty? || blue_green_terminate_inactive
+          active_ebenv.log("applying inactive settings...")
+          active_ebenv.apply_settings(inactive_settings)
+        end
+      end
+
+      private
+      def active_ebenv?(ebenv)
+        ebenv.cname_prefix == @component.cname_prefix
+      end
+
+      def ebenvs
+        [ebenv('a'), ebenv('b')]
+      end
+
+      def ebenv(suffix, cname_prefix=nil)
+        @component.new_eb_env(suffix, cname_prefix || inactive_cname_prefix)
+      end
+
+      def inactive_cname_prefix
+        "#{@component.cname_prefix}-inactive"
+      end
+    end
+  end
+end