man_eb_deployer 0.8.0

data/lib/generators/eb_deployer/install/templates/eb_deployer.yml.erb

@@ -0,0 +1,181 @@
+# application name
+application: <%= app_name %>
+
+# common settings for all environments
+common:
+  # Solution stack for elastic beanstalk, default is 64bit tomcat 7 for JAVA app
+  solution_stack_name: <%= solution_stack_name %>
+
+  # Tier name for environments. Current supported values are WebServer and Worker
+  # tier: WebServer
+
+  # AWS region to deploy. Default to us-east-1
+  # region: us-west-1
+
+  # There are two deployment strategies: 'blue-green' or 'inplace-update'.
+  # Blue green deployments keep two elastic beanstalk environments and always deploy to
+  # inactive one, to achieve zero downtime.
+  # Inplace-update strategy will only keep one environment, and update the version inplace on
+  # deploy. Inplace-update will save resources but will suffer from downtime.
+  # (All old environments need be destroyed when you switching between strategies.)
+  # Default strategy is 'blue-green'.
+  # strategy: blue-green
+
+  # Name of s3 bucket where uploaded application packages will be stored.
+  # Note that the string ".packages" will be added as a suffix to your bucket.
+  # So, if "thoughtworks.simple" is passed as the bucket name, the actual s3 bucket
+  # name will be thoughtworks.simple.packages. Default to application name.
+  # package_bucket: my-s3-bucket
+
+  # If phoenix mode is turned 'on', it will terminate the old elastic
+  # beanstalk environment and recreate a new one on deploy. For blue-green
+  # deployment it will terminate the inactive environment first then
+  # recreate it. This is useful to avoid configuration drift and
+  # accumulating state on the ec2 instances. Also it has the benefit of
+  # keeping your ec2 instance system package upto date, because everytime ec2
+  # instance boots up from AMI it does a system update. Default is 'off' but we suggest
+  # you override it to 'on' for production environment.
+  # phoenix_mode: false
+
+  # Terminates the inactive stack upon successful blue-green deployment.
+  # This is useful if you require HA deployment but don't want the cost of running
+  # a stack that is not in use. See https://github.com/ThoughtWorksStudios/eb_deployer/pull/44/
+  # for other strategies that can be employed.
+  # Note: As EB Deployer uses DNS to switch between stacks, this should be used only if you
+  #       both put in a sufficient wait period and accept the risk that traffic may still be
+  #       going to the inactive stack when it is terminated, due to client DNS caching, for example.
+  #blue_green_terminate_inactive: false
+
+  # How long to wait before terminating the inactive stack if 'blue_green_terminate_inactive' is set to true.
+  #blue_green_terminate_inactive_wait: 600
+
+  # How often to check for changes to the inactive stack during the 'blue_green_terminate_inactive_wait' period.
+  #blue_green_terminate_inactive_sleep: 15
+
+  # Specifies the maximum number of versions to keep. Older versions are removed
+  # and deleted from the S3 source bucket as well. If specified as zero or not
+  # specified, all versions will be kept.  If a version_prefix is given, only removes
+  # version starting with the prefix.
+  # keep_latest: 200
+
+  # Specifies a prefix to prepend to the version label.
+  # This can be useful if you want to use different binaries for different
+  # environments.
+  # version_prefix:
+
+  # Generating version label for package to be deployed. A readable version label will
+  # provide better traceablity of your deployment process.
+  # By default setting is:
+  # version_label: <%%= package_digest %>
+  # which means using MD5 digest of the package file. If you deploy using build
+  # pipeline tool such as GO, switching to pipline counter is highly suggested to
+  # increase the readability. Following example will read pipeline counter from environment
+  # variable with a fall back to digest for local deployment:
+  # version_label: <%%= ENV['GO_PIPELINE_COUNTER'] || package_digest %>
+
+
+  # Smoke test value should be a piece of ruby code with access to single variable
+  # "host_name" -- environment DNS name. Smoke test snippet will be evaluated at
+  # the end of the deployment for inplace-update deployment. For blue-green
+  # deployment it will run after inactive environment update is completed and before
+  # switching over.
+  # Defining a smoke test is highly recommended for serious usage. By default we use
+  # The simplest one that just be checking server landing page using curl, e.g.
+  smoke_test: |
+    curl_http_code = "curl -s -o /dev/null -w \"%{http_code}\" http://#{host_name}"
+    Timeout.timeout(600) do
+      until ['200', '301', '302'].include?(`#{curl_http_code}`.strip)
+        sleep 5
+      end
+    end
+
+  # List of Elastic Beanstalk health states that will be considered healthy when deploying.
+  # You may want to override default 'Green' with that list, if for example you are using
+  # Elastic Beanstalk worker tier and you don't want eb_deployer to fail if you have messages
+  # in your SQS queue.
+  # By default eb_deployer only considers 'Green' as the healthy state.
+  # For a list of all Elastic Beanstalk health states refer to:
+  # http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features.healthstatus.html#using-features.healthstatus.colors
+  # accepted_healthy_states:
+  #  - Green
+  #  - Yellow
+
+  # Elastic Beanstalk settings that will apply to the environments you are
+  # deploying.
+  # For all available options take a look at
+  # http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options.html
+  option_settings:
+    # Following is an example of set EC2 ssh key name. This allow you ssh into the ec2
+    # instance. See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html
+    # - namespace: aws:autoscaling:launchconfiguration
+    #   option_name: EC2KeyName
+    #   value: <my-ec2-key-name>
+
+    # Following is an example which changes EC2 instance type
+    # from t1.micro (default) to m1.small. Instances with t1.micro type sometime
+    # are not very responsible, so m1.small is suggested for saving time.
+    # But if you care about the marginal cost difference you can comment this out to
+    # go back to t1.micro.
+    - namespace: aws:autoscaling:launchconfiguration
+      option_name: InstanceType
+      value: m1.small
+    - namespace: aws:elasticbeanstalk:application:environment
+      option_name: DATABASE_NAME
+      value: <%= alphanumeric_name %>
+    - namespace: aws:elasticbeanstalk:application:environment
+      option_name: DATABASE_USERNAME
+      value: <%= alphanumeric_name %>
+    - namespace: aws:elasticbeanstalk:application:environment
+      option_name: DATABASE_PASSWORD
+      value: <%= db_password %>
+
+
+  # If resources specified, eb_deployer will use the CloudFormation
+  # template you provide to create a default CloudFormation stack with
+  # name <application_name>-<env-name> for the environment current
+  # deploying. And Outputs of the CloudFormation can be mapped to Elastic Beanstalk
+  # options settings.
+  # keys:
+  #    template => CloudFormation template file with JSON format
+  #    policy => CloudFormation policy file with JSON format
+  #    override_policy => (false) If override_policy is true and a policy file is provided then the policy will temporarily override any existing policy on the resource stack during this update,
+  #                       otherwise the provided policy will replace any existing policy on the resource stack
+  #    inputs => A Hash, input values for the CloudFormation template
+  #    outputs => A Hash with key map to your CloudFormation template outputs and value as elastic beanstalk settings namespace and option_name.
+  #    capabilities => An array. You need set it to ['CAPABILITY_IAM'] if the
+  # template include IAM Instance Profile.
+  resources:
+    # Creating a RDS instance by CloudFormation, so that database will not hook up with any ElasticBeanstalk environment
+    template: config/rds.json
+    inputs:
+      DBName: <%= alphanumeric_name %>
+      DBUser: <%= alphanumeric_name %>
+      DBPassword: <%= db_password %>
+    outputs:
+      RDSPassSecurityGroup:
+        namespace: aws:autoscaling:launchconfiguration
+        option_name: SecurityGroups
+      RDSHost:
+        namespace: aws:elasticbeanstalk:application:environment
+        option_name: DATABASE_HOST
+      RDSPort:
+        namespace: aws:elasticbeanstalk:application:environment
+        option_name: DATABASE_PORT
+
+# You can define environment here. Each environment can overriden any common settings
+environments:
+  dev:
+    # overriding common settings
+    strategy: inplace-update
+  production:
+    option_settings:
+      # example for overriding common option_settings: providing least redundancy
+      # in production environment.
+      # - namespace: aws:autoscaling:asg
+      #   option_name: MinSize
+      #    value: "2"
+
+      # For Rails4: config/secrets.yml
+      - namespace: aws:elasticbeanstalk:application:environment
+        option_name: SECRET_KEY_BASE
+        value: <%= secure_random(128) %>

data/lib/generators/eb_deployer/install/templates/ebextensions/01_postgres_packages.config

@@ -0,0 +1,5 @@
+---
+packages:
+  yum:
+    postgresql93-libs: []
+    postgresql93-devel: []

data/lib/generators/eb_deployer/install/templates/postgres_rds.json

@@ -0,0 +1,88 @@
+{
+    "Outputs": {
+        "RDSHost": {
+            "Description": "Database endpoint address",
+            "Value": { "Fn::GetAtt": ["RDSDatabase", "Endpoint.Address"] }
+        },
+        "RDSPort": {
+            "Description": "Database endpoint port",
+            "Value": { "Fn::GetAtt": ["RDSDatabase", "Endpoint.Port"] }
+        },
+        "RDSPassSecurityGroup": {
+            "Description": "Security group assign to ec2 instance that need access to rds instance",
+            "Value": {
+                "Ref": "RDSPassSecurityGroup"
+            }
+        }
+    },
+
+    "Parameters": {
+        "DBUser": {
+            "NoEcho": "false",
+            "Description": "The name of master user for the client DB Instance.",
+            "Type": "String",
+            "ConstraintDescription": "must begin with a letter and contain only alphanumeric characters"
+        },
+
+        "DBName": {
+            "NoEcho": "false",
+            "Description": "The DB Name of the RDS instance",
+            "Type": "String",
+            "ConstraintDescription": "must contain only alphanumeric characters"
+        },
+
+        "DBPassword": {
+            "NoEcho": "true",
+            "Description": "The master password for the DB instance.",
+            "Type": "String",
+            "ConstraintDescription": "must contain only alphanumeric characters"
+        }
+    },
+
+    "Resources": {
+        "RDSDBSecurityGroup": {
+            "Type": "AWS::RDS::DBSecurityGroup",
+            "Properties": {
+                "GroupDescription": "Enable database access to Beanstalk application",
+                "DBSecurityGroupIngress": {
+                    "EC2SecurityGroupName": {
+                        "Ref": "RDSPassSecurityGroup"
+                    }
+                }
+            }
+        },
+
+        "RDSDatabase": {
+            "Type": "AWS::RDS::DBInstance",
+            "DeletionPolicy": "Delete",
+            "Properties": {
+                "MasterUsername": {
+                    "Ref": "DBUser"
+                },
+                "DBSecurityGroups": [
+                    {
+                        "Ref": "RDSDBSecurityGroup"
+                    }
+                ],
+                "DBInstanceClass": "db.m1.small",
+                "AllocatedStorage": "5",
+                "MultiAZ": "false",
+                "EngineVersion": "9.3.3",
+                "DBName": {
+                    "Ref": "DBName"
+                },
+                "MasterUserPassword": {
+                    "Ref": "DBPassword"
+                },
+                "Engine": "postgres"
+            }
+        },
+
+        "RDSPassSecurityGroup": {
+            "Type": "AWS::EC2::SecurityGroup",
+            "Properties": {
+                "GroupDescription": "SecurityGroup access RDS database."
+            }
+        }
+    }
+}

data/test/aws_driver_stubs.rb

@@ -0,0 +1,350 @@
+class EBStub
+  attr_reader :envs, :events
+  def initialize
+    @apps = []
+    @envs = {}
+    @versions = {}
+    @envs_been_deleted = {}
+    @versions_deleted = {}
+    @event_fetched_times = 0
+    @envs_health_states = {}
+    @events = nil
+  end
+
+  def create_application(app)
+    raise 'already exists' if application_exists?(app)
+    @apps << app
+  end
+
+  def delete_application(app)
+    raise "there are environments running for this app" unless environment_names_for_application(app).empty?
+    @apps.delete(app)
+  end
+
+  def application_exists?(app)
+    @apps.include?(app)
+  end
+
+  def create_environment(app, env, solution_stack, cname_prefix, version, tier, tags, settings, template_name)
+    raise 'cname prefix is not avaible' if @envs.values.detect { |value| value[:cname_prefix] == cname_prefix }
+    raise "env name #{env} is longer than 23 chars" if env.size > 23
+    raise "app not exists" unless application_exists?(app)
+    @envs[env_key(app, env)] = {
+      :name => env,
+      :application => app,
+      :solution_stack => solution_stack,
+      :version => version,
+      :cname_prefix => cname_prefix,
+      :tier => tier,
+      :tags => tags,
+      :settings => settings,
+      :template_name => template_name }
+    set_env_ready(app, env, false)
+  end
+
+  def delete_environment(app, env)
+    @envs.delete(env_key(app, env))
+    @envs_been_deleted[app] ||= []
+    @envs_been_deleted[app] << env
+  end
+
+  def update_environment_settings(app, env, settings)
+    raise "not in ready state, consider waiting for previous action finish by pulling envents" unless env_ready?(app, env)
+    @envs[env_key(app, env)].merge!(:settings => settings)
+  end
+
+
+  def update_environment(app, env, version, tier, settings, template_name)
+    raise "not in ready state, consider waiting for previous action finish by pulling envents" unless env_ready?(app, env)
+    @envs[env_key(app, env)].merge!(:version => version, :settings => settings, :tier => tier, :template_name => template_name)
+    set_env_ready(app, env, false)
+  end
+
+  def environment_exists?(app_name, env_name)
+    @envs.has_key?(env_key(app_name, env_name))
+  end
+
+  def create_application_version(app_name, version_label, source_bundle)
+    @versions[app_name] ||= []
+    @versions[app_name] << {
+      :version_label => version_label,
+      :source_bundle => source_bundle,
+      :date_created => Time.now,
+      :date_updated => Time.now
+    }
+  end
+
+  def delete_application_version(app_name, version, delete_source_bundle)
+    @versions_deleted[app_name] ||= []
+    @versions_deleted[app_name] << version
+    @versions[app_name].delete_if { |apv| apv[:version_label] == version }
+  end
+
+  def application_versions(app_name)
+    @versions[app_name]
+  end
+
+  def application_version_labels(app_name)
+    if @versions[app_name]
+      @versions[app_name].map { |appv| appv[:version_label]}
+    else
+      []
+    end
+  end
+
+  # simulating elasticbeanstalk events api behavior
+  #  use set_events and append_events to set fake events
+  def fetch_events(app_name, env_name, options={})
+    @event_fetched_times += 1
+    set_env_ready(app_name, env_name, true) # assume env become ready after it spit out all the events
+
+    unless @events # unrestricted mode for testing if no explicit events set
+      return [generate_event_from_messages(['Successfully deployed new configuration to environment',
+                                           'terminateEnvironment completed successfully',
+                                           'Successfully launched environment',
+                                           'Completed swapping CNAMEs for environments'
+                                          ], Time.now + @event_fetched_times), nil]
+    end
+
+    events = @events[env_key(app_name, env_name)][@event_fetched_times - 1]
+
+    if options.has_key?(:start_time)
+      start_time = Time.parse(options[:start_time])
+      events = events.select { |e| e[:event_date] >= start_time  }
+    end
+
+    if limit = options[:max_records]
+      events = events[0..limit]
+    end
+
+    [events, nil]
+  end
+
+  # add fake events for each times of fetch events call
+  # message passed in should be old to new order
+  # e.g.  given set_events("myapp", "test", ['a'], ['b', 'c'])
+  # then
+  #    fetch_events('myapp', 'test') # => [{message: 'a'}]  for first time
+  #    fetch_events('myapp', 'test') # => [{message: 'c'}, {message: 'b'}, {message: 'a'}] for the second time call
+  def set_events(app_name, env_name, *messages)
+    events_seq = []
+    messages.each do |messages_for_call_seq|
+      if old_events = events_seq.last
+        last_event_date = old_events.first && old_events.first[:event_date]
+        events_seq << (generate_event_from_messages(messages_for_call_seq, last_event_date) + old_events)
+      else
+        events_seq << generate_event_from_messages(messages_for_call_seq)
+      end
+    end
+    @events ||= {}
+    @events[env_key(app_name, env_name)] = events_seq
+  end
+
+  def environment_cname_prefix(app_name, env_name)
+    return unless @envs[env_key(app_name, env_name)]
+    @envs[env_key(app_name, env_name)][:cname_prefix]
+  end
+
+  def environment_cname(app_name, env_name)
+    return unless @envs[env_key(app_name, env_name)]
+    if cname_prefix = environment_cname_prefix(app_name, env_name)
+      cname_prefix + ".us-west-1.elasticbeanstalk.com"
+    end
+  end
+
+
+  def environment_swap_cname(app_name, env1_name, env2_name)
+    raise "#{env1_name} not in ready state, consider to wait for previous action finsish by pulling events" unless env_ready?(app_name, env1_name)
+
+    env1, env2 = @envs[env_key(app_name, env1_name)], @envs[env_key(app_name, env2_name)]
+    temp = env1[:cname_prefix]
+    env1[:cname_prefix] = env2[:cname_prefix]
+    env2[:cname_prefix] = temp
+    set_env_ready(app_name, env1_name, false)
+    set_env_ready(app_name, env2_name, false)
+  end
+
+  def environment_status(app_name, env_name)
+    'ready'
+  end
+
+  def environment_health_state(app_name, env_name)
+    @envs_health_states.fetch(app_name+env_name, 'Green')
+  end
+
+  def environment_verion_label(app_name, env_name)
+    @envs[env_key(app_name, env_name)][:version]
+  end
+
+  def list_solution_stack_names
+    ["64bit Amazon Linux 2014.09 v1.1.0 running Tomcat 7 Java 7"]
+  end
+
+  #test only
+
+  def mark_env_health_state_as(app_name, env_name, state)
+    @envs_health_states[app_name+env_name] = state
+  end
+
+  def mark_all_envs_ready
+    @envs.values.each { |env| set_env_ready(env[:application], env[:name], true) }
+  end
+
+  def set_solution_stacks(names)
+    @solution_stacks = names
+  end
+
+  def environment_tier(app_name, env_name)
+    @envs[env_key(app_name, env_name)][:tier]
+  end
+
+  def environment_tags(app_name, env_name)
+    @envs[env_key(app_name, env_name)][:tags]
+  end
+
+  def environment_settings(app_name, env_name)
+    @envs[env_key(app_name, env_name)][:settings]
+  end
+
+  def environment_names_for_application(app)
+    @envs.inject([]) do |memo, pair|
+      _, env = pair
+      memo << env[:name] if env[:application] == app
+      memo
+    end
+  end
+
+  def environments_been_deleted(app)
+    @envs_been_deleted[app] || []
+  end
+
+  def versions_deleted(app_name)
+    @versions_deleted[app_name]
+  end
+
+  def template_name(app_name, env_name)
+    @envs[env_key(app_name, env_name)][:template_name]
+  end
+
+  private
+
+  def set_env_ready(app, env, ready)
+    if record = @envs[env_key(app, env)]
+      record[:ready] = ready
+    end
+  end
+
+  def env_ready?(app, env)
+    @envs[env_key(app, env)][:ready]
+  end
+
+  def generate_event_from_messages(messages, start_time=Time.now)
+    start_time ||= Time.now
+    events = messages.map do |m|
+      { :message => m }
+    end
+
+    events.each_with_index do |e, i|
+      e[:event_date] = start_time + (i + 1)
+    end
+
+    events.reverse
+  end
+
+  def env_key(app, name)
+    [app, name].join("-")
+  end
+
+end
+
+class S3Stub
+  def initialize
+    @buckets = {}
+  end
+
+  def create_bucket(bucket_name)
+    @buckets[bucket_name] = {}
+  end
+
+  def bucket_exists?(bucket_name)
+    @buckets.has_key?(bucket_name)
+  end
+
+  def objects(bucket_name)
+    @buckets[bucket_name]
+  end
+
+  def object_length(bucket_name, obj_name)
+    @buckets[bucket_name][obj_name] && File.size(@buckets[bucket_name][obj_name])
+  end
+
+  def upload_file(bucket_name, obj_name, file)
+    @buckets[bucket_name][obj_name] = file
+  end
+
+end
+
+class CFStub
+  def initialize
+    @stacks = {}
+    @events = {}
+    @event_fetch_counter = 0;
+  end
+
+  def create_stack(name, template, opts)
+    @stacks[name] = {:template => template, :opts => opts }
+  end
+
+  def update_stack(name, template, opts)
+    @stacks[name] = @stacks[name].merge(:template => template, :opts => opts)
+  end
+
+  def stack_status(name)
+    @stacks[name] && :update_complete
+  end
+
+  def stack_exists?(name)
+    @stacks.has_key?(name)
+  end
+
+  def query_output(name, key)
+    raise Aws::CloudFormation::Errors::ValidationError.new(nil, nil) unless stack_exists?(name)
+    "value of #{key}"
+  end
+
+  def stack_config(name)
+    @stacks[name][:opts]
+  end
+
+  def set_events(name, *messages)
+    events_seq = []
+    messages.each do |messages_for_call_seq|
+      if old_events = events_seq.last
+        events_seq << generate_event_from_messages(name, messages_for_call_seq) + old_events
+      else
+        events_seq << generate_event_from_messages(name, messages_for_call_seq)
+      end
+    end
+    @events[name] = events_seq
+  end
+
+  def fetch_events(name, opts={})
+    @event_fetch_counter += 1
+    if es = @events[name]
+      return es[@event_fetch_counter - 1], nil
+    else
+      return generate_event_from_messages(name, ["UPDATE_COMPLETE"]), nil
+    end
+  end
+
+  private
+
+  def generate_event_from_messages(stack, messages)
+    messages.map do |message|
+      OpenStruct.new(timestamp: Time.now,
+                     resource_type: 'AWS::CloudFormation::Stack',
+                     logical_resource_id: stack,
+                     resource_status: message)
+    end.reverse
+  end
+end

data/test/beanstalk_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+class BeanstalkTest < Test::Unit::TestCase
+
+  class StubbedCnameResponder
+    def initialize(stubbed_response)
+      @stubbed_response = stubbed_response
+    end
+
+    def describe_environments(options)
+      {:environments => [@stubbed_response.merge(:status => "online")]}
+    end
+  end
+
+  def test_cname_prefix_will_parse_both_legacy_and_regionalized_domains
+    legacy_domain = "mingle-saas.elasticbeanstalk.com"
+    regionalized_domain = "mingle-saas.us-west-1.elasticbeanstalk.com"
+
+    assert_equal "mingle-saas", EbDeployer::AWSDriver::Beanstalk.new(StubbedCnameResponder.new(:cname => legacy_domain)).environment_cname_prefix("mingle", "saas")
+    assert_equal "mingle-saas", EbDeployer::AWSDriver::Beanstalk.new(StubbedCnameResponder.new(:cname => regionalized_domain)).environment_cname_prefix("mingle", "saas")
+  end
+
+end