cf_deployer 1.2.8

data/lib/cf_deployer/application.rb

@@ -0,0 +1,74 @@
+module CfDeployer
+  class Application
+    attr_reader :components
+
+    def initialize(context = {})
+     @context = context
+     get_components
+     add_component_dependencies
+     @components.sort!
+    end
+
+    def get_components
+      @components = []
+      @context[:components].keys.each do | key |
+        component = Component.new(@context[:application], @context[:environment], key.to_s, @context[:components][key])
+        @components << component
+      end
+    end
+
+    def add_component_dependencies
+      @context[:components].keys.each do | key |
+        component = @components.find { |c| c.name == key.to_s }
+        dependencies = @context[:components][key][:'depends-on'] || []
+        dependencies.each do | parent_name |
+          parent = @components.find { |c| c.name == parent_name }
+          if parent
+            parent.children << component
+            component.dependencies << parent
+          end
+        end
+      end
+    end
+
+    def deploy
+      Log.debug @context
+      components = get_targets().sort
+      components.each &:deploy
+    end
+
+    def json
+      components = get_targets().sort
+      components.each &:json
+    end
+
+    def status component_name, verbosity
+      statuses = {}
+      @components.select { |component|  component_name.nil? || component_name == component.name }.each do |component|
+        statuses[component.name] = component.status(verbosity != 'stacks')
+      end
+      statuses
+    end
+
+    def destroy
+      components = get_targets.sort { |a, b| b <=> a }
+      components.each &:destroy
+    end
+
+    def kill_inactive
+      component = get_targets.first
+      component.kill_inactive
+    end
+
+    def switch
+      @context[:targets].each do | component_name |
+        @components.find { |c| c.name == component_name }.switch
+      end
+    end
+
+    private
+    def get_targets
+      targets = @components.select { |c| @context[:targets].include?(c.name) }
+    end
+  end
+end

data/lib/cf_deployer/application_error.rb

@@ -0,0 +1,4 @@
+module CfDeployer
+  class ApplicationError < StandardError
+  end
+end

data/lib/cf_deployer/aws_constants.rb

@@ -0,0 +1,3 @@
+module AwsConstants
+  ELB_RESOURCE_TYPE = 'AWS::ElasticLoadBalancing::LoadBalancer'
+end

data/lib/cf_deployer/cli.rb

@@ -0,0 +1,111 @@
+require 'thor'
+
+module CfDeployer
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    class_option :'config-file', :aliases => '-f', :desc => "cf_deployer config file", :default => 'config/cf_deployer.yml'
+    class_option :'log-level',   :aliases => '-l', :desc => "logging level", :enum => %w(info debug aws-debug), :default => 'info'
+    class_option :'dry-run',     :aliases => '-d', :desc => "Say what we would do but don't actually make changes to anything", :banner => ''
+    class_option :settings,      :aliases => '-s', :type => :hash, :desc =>  "key:value pair to overwrite setting in config"
+    class_option :inputs,        :aliases => '-i', :type => :hash, :desc => "key:value pair to overwrite in template inputs"
+    class_option :region,        :aliases => '-r', :desc => "Amazon region", :default => 'us-east-1'
+
+    desc "deploy [ENVIRONMENT] [COMPONENT]\t", 'Deploy the specified components'
+    def deploy environment, component = nil
+      prep_for_action :deploy, environment, component
+      CfDeployer.deploy merged_options
+    end
+
+    desc "destroy [ENVIRONMENT] [COMPONENT]\t", 'Destroy the specified environment/component'
+    def destroy environment, component = nil
+      prep_for_action :destroy, environment, component
+      CfDeployer.destroy merged_options
+    end
+
+    desc "config [ENVIRONMENT]", 'Show parsed config'
+    def config environment
+      prep_for_action :config, environment
+      CfDeployer.config merged_options
+    end
+
+    desc "json [ENVIRONMENT] [COMPONENT]", 'Show parsed CloudFormation JSON for the target component'
+    def json environment, component = nil
+      prep_for_action :json, environment, component
+      CfDeployer.json merged_options
+    end
+
+    desc 'status [ENVIRONMENT] [COMPONENT]', 'Show the status of the specified Cloud Formation components specified in your yml'
+    method_option :verbosity,  :aliases => '-v', :desc => 'Verbosity level',  :enum => ['stacks','instances','all'], :default => 'instances'
+    method_option :'output-format', :aliases => '-o', :enum => ['human','json'], :default => 'human', :desc => 'Output format'
+    def status environment, component = nil
+      prep_for_action :status, environment, component
+      CfDeployer.status merged_options
+    end
+
+    desc 'kill_inactive [ENVIRONMENT] [COMPONENT]', 'Destroy the inactive stack for a given component/environment'
+    def kill_inactive environment, component
+      prep_for_action :kill_inactive, environment, component
+      CfDeployer.kill_inactive merged_options
+    end
+
+    desc 'switch [ENVIRONMENT] [COMPONENT]', 'Switch active and inactive stacks'
+    def switch environment, component
+      prep_for_action :switch, environment, component
+      CfDeployer.switch merged_options
+    end
+
+    no_commands do
+
+      def prep_for_action action, environment, component = nil
+        no_component_required_actions = [:config, :status]
+        if environment == 'help' || (no_component_required_actions.include? action && component == nil)
+          self.class.command_help shell, action
+          exit 0
+        end
+        @environment = environment
+        @component = [component].compact
+        validate_cli options
+        set_log_level
+        detect_dry_run
+      end
+
+      def merged_options
+        symbolize_all_keys options.merge({:environment => @environment, :component => @component})
+      end
+
+      def set_log_level
+        if options[:'log-level'] == 'aws-debug'
+          CfDeployer::Log.level = 'debug'
+          AWS.config :logger => Logger.new($stdout)
+        else
+          CfDeployer::Log.level = options[:'log-level']
+        end
+      end
+
+      def detect_dry_run
+        CfDeployer::Driver::DryRun.enable if options[:'dry-run']
+      end
+
+      def validate_cli cli_options
+        unless File.file?(options[:'config-file'])
+          error_exit "ERROR:  #{options[:'config-file']} is not a file."
+        end
+        error_exit "ERROR:  No environment specified!" unless @environment
+      end
+
+      def symbolize_all_keys(hash)
+        return hash unless hash.is_a?(Hash)
+        hash.inject({}){|memo,(k,v)|  memo.delete(k); memo[k.to_sym] = symbolize_all_keys(v); memo}
+      end
+
+      def error_exit message
+        puts message
+        exit 1
+      end
+
+    end
+  end
+end

data/lib/cf_deployer/component.rb

@@ -0,0 +1,103 @@
+module CfDeployer
+  class Component
+    attr_reader :name, :dependencies, :children
+
+    def initialize(application_name, environment_name, component_name, context)
+      @application_name = application_name
+      @environment_name = environment_name
+      @name = component_name
+      @context = context
+      @dependencies = []
+      @children = []
+      Log.debug "initializing #{name}.."
+      Log.debug @context
+    end
+
+    def exists?
+       strategy.exists?
+    end
+
+    def kill_inactive
+      strategy.kill_inactive
+    end
+
+    def deploy
+      Log.debug "deploying #{name}..."
+      @dependencies.each do |parent|
+        parent.deploy unless(parent.exists?)
+      end
+      resolve_settings
+      strategy.deploy
+    end
+
+    def json
+      resolve_settings
+      puts "#{name} json template:"
+      puts ConfigLoader.component_json(name, @context)
+    end
+
+    def destroy
+      raise ApplicationError.new("Unable to destroy #{name}, it is depended on by other components") if any_children_exist?
+      strategy.destroy
+    end
+
+    def switch
+      exists? ? strategy.switch : (raise ApplicationError.new("No stack exists for component: #{name}"))
+    end
+
+
+    def output_value(key)
+      strategy.output_value(key)
+    end
+
+    def <=>(other)
+      i_am_depednent = depends_on? other
+      it_is_dependent = other.depends_on? self
+
+      if i_am_depednent
+        1
+      elsif it_is_dependent
+        -1
+      else
+        0
+      end
+    end
+
+    def inspect
+      "component: #{name}"
+    end
+
+    def depends_on?(component, source=self)
+      raise ApplicationError.new("Cyclic dependency") if @dependencies.include?(source)
+      @dependencies.include?(component) || @dependencies.any? { |d| d.depends_on?(component, source) }
+    end
+
+    def status get_resource_statuses
+      strategy.status get_resource_statuses
+    end
+
+    private
+
+    def resolve_settings
+      inputs.each do |key, value|
+        if(value.is_a? Hash)
+          dependency = @dependencies.find { |d| d.name == value[:component] }
+          output_key = value[:'output-key']
+          inputs[key] = dependency.output_value(output_key)
+        end
+      end
+    end
+
+    def any_children_exist?
+      children.any?(&:exists?)
+    end
+
+    def inputs
+      @context[:inputs]
+    end
+
+    def strategy
+      @strategy ||= DeploymentStrategy.create(@application_name, @environment_name, name, @context)
+    end
+  end
+end

data/lib/cf_deployer/config_loader.rb

@@ -0,0 +1,189 @@
+module CfDeployer
+  class ConfigLoader
+
+    def self.component_json component, config
+      json_file = File.join(config[:config_dir], "#{component}.json")
+      raise ApplicationError.new("#{json_file} is missing") unless File.exists?(json_file)
+      ERB.new(File.read(json_file)).result(binding)
+    end
+
+    def load(options)
+      config_text = File.read(options[:'config-file'])
+      erbed_config = erb_with_environment_and_region(config_text, options[:environment], options[:region])
+      yaml = symbolize_all_keys(load_yaml(erbed_config))
+      @config = options.merge(yaml)
+      @config[:components] ||= {}
+      @config[:settings] ||= {}
+      @config[:environments] ||= {}
+      @config[:tags] ||= {}
+      get_targets
+      copy_config_dir
+      merge_section(:settings)
+      merge_section(:inputs)
+      merge_section(:tags)
+      copy_region_app_env_component
+      get_cf_template_keys('Parameters')
+      get_cf_template_keys('Outputs')
+      set_default_settings
+      @config.delete(:settings)
+      @config
+    end
+
+    private
+
+    def load_yaml(text)
+      YAML.load text
+    rescue
+      raise ApplicationError.new("The config file is not a valid yaml file")
+    end
+
+
+    def merge_section(section)
+      merge_component_options section
+      merge_environment_options(@config[:environment], section)
+      merge_environment_variables section
+      @config[:cli_overrides] ||= {}
+      merge_options(@config[:cli_overrides][section] || {}, section)
+    end
+
+    def get_targets
+      @config[:component] ||= []
+      @config[:targets] = @config[:component].length == 0 ? @config[:components].keys.map(&:to_s) : @config[:component]
+    end
+
+    def copy_region_app_env_component
+      @config[:components].each do |component_name, component|
+        component[:settings][:region]          = @config[:region]
+        component[:inputs][:region]            = @config[:region]
+
+        component[:settings][:application]          = @config[:application]
+        component[:inputs][:application]            = @config[:application]
+
+        component[:settings][:component]            = component_name.to_s
+        component[:inputs][:component]              = component_name.to_s
+
+        component[:settings][:environment]          = @config[:environment]
+        component[:inputs][:environment]            = @config[:environment]
+      end
+    end
+
+    def set_default_settings
+      @config[:components].each do |component_name, component|
+        if  component[:'deployment-strategy'] == 'cname-swap'
+          component[:settings][:'elb-name-output'] ||= Defaults::ELBName
+          component[:settings][:'dns-driver'] ||= Defaults::DNSDriver
+        end
+        component[:settings][:'raise-error-for-unused-inputs'] ||= Defaults::RaiseErrorForUnusedInputs
+        component[:settings][:'auto-scaling-group-name-output'] ||= [Defaults::AutoScalingGroupName] if component[:'deployment-strategy'] == 'auto-scaling-group-swap'
+        component[:settings][:'auto-scaling-group-name-output'] ||= [Defaults::AutoScalingGroupName] if component[:'defined_outputs'].keys.include?(Defaults::AutoScalingGroupName.to_sym)
+        if component[:settings][:'keep-previous-stack'] == nil
+          component[:settings][:'keep-previous-stack'] = Defaults::KeepPreviousStack
+        end
+      end
+    end
+
+    def get_cf_template_keys(name)
+      @config[:components].keys.each do |component|
+        parameters = cf_template(component)[name] || {}
+        @config[:components][component]["defined_#{name.downcase}".to_sym] = symbolize_all_keys(parameters)
+      end
+    end
+
+    def cf_template(component)
+      config =  deep_dup(@config[:components][component])
+      config[:inputs].each do |key, value|
+        if value.is_a?(Hash)
+          output_key = value[:'output-key']
+          config[:inputs][key] = "#{value[:component]}::#{output_key}"
+        end
+      end
+
+      json_content = self.class.component_json component.to_s, config
+      begin
+        JSON.load json_content
+      rescue JSON::ParserError => e
+        puts json_content
+        puts '=' * 80
+        puts e.message[0..300]
+        puts '=' * 80
+        raise "Couldn't parse JSON for component #{component}"
+      end
+    end
+
+    def config_dir
+      File.dirname(@config[:'config-file'])
+    end
+
+    def copy_config_dir
+      @config[:components].each do |component_name, component|
+        component ||= {}
+        @config[:components][component_name] = component
+        component[:config_dir] = config_dir
+      end
+    end
+
+    def merge_component_options(section)
+      common_options = @config[section] || {}
+      @config[:components].keys.each do |component|
+        @config[:components][component] ||= {}
+        component_options = @config[:components][component].delete(section) || {}
+        @config[:components][component][section] = common_options.merge(component_options)
+      end
+    end
+
+    def merge_environment_options(environment_name, section)
+      return unless environment_name
+      environment = @config[:environments][environment_name.to_sym] || {}
+      environment_options = environment[section] || {}
+      merge_options(environment_options, section)
+      environment_components = environment[:components] || {}
+      merge_environment_component(environment_components, section)
+    end
+
+    def merge_options(options, section)
+      @config[:components].keys.each do |component|
+        component_options = @config[:components][component].delete(section) || {}
+        @config[:components][component][section] = component_options.merge(options)
+      end
+    end
+
+    def merge_environment_component(environment_components, section)
+      environment_components.keys.each do |component|
+        @config[:components][component] ||= {}
+        component_options = @config[:components][component].delete(section) || {}
+        @config[:components][component][section] = component_options.merge(environment_components[component][section] || {})
+      end
+    end
+
+    def merge_environment_variables(section)
+      @config[:components].keys.each do |component|
+        merge_environment_variables_to_options( @config[:components][component][section], section)
+      end
+    end
+
+    def merge_environment_variables_to_options(options, section)
+      options.keys.each do |key|
+        environment_variable = ENV["cfdeploy_#{section}_#{key.to_s}"]
+        options[key] = environment_variable if environment_variable
+      end
+    end
+
+    def symbolize_all_keys(hash)
+      return hash unless hash.is_a?(Hash)
+      hash.inject({}){|memo,(k,v)| memo.delete(k); memo[k.to_sym] = symbolize_all_keys(v); memo}
+    end
+
+    def erb_with_environment_and_region(contents, environment, region)
+      ERB.new(contents).result(binding)
+    end
+
+    def deep_dup(hash)
+      new_hash = {}
+      hash.each do |key, value|
+        value.is_a?(Hash) ? new_hash[key] = deep_dup(value) : new_hash[key] = value
+      end
+      new_hash
+    end
+
+  end
+end

data/lib/cf_deployer/config_validation.rb

@@ -0,0 +1,138 @@
+module CfDeployer
+  class ConfigValidation
+
+    class ValidationError < ApplicationError
+    end
+
+    CommonInputs = [:application, :environment, :component, :region]
+    EnvironmentOptions = [:settings, :inputs, :tags, :components]
+    ComponentOptions = [:settings, :inputs, :tags, :'depends-on', :'deployment-strategy', :'before-destroy', :'after-create', :'after-swap', :'defined_outputs', :'defined_parameters', :config_dir, :capabilities]
+
+    def validate config, validate_inputs = true
+      @config = config
+      @errors = []
+      @warnings = []
+      check_application_name
+      check_components validate_inputs
+      check_environments
+      @warnings.each { |message| puts "WARNNING:#{message}" }
+      raise ValidationError.new(@errors.join("\n")) if @errors.length > 0
+    end
+
+    private
+
+
+    def check_asg_name_output(component)
+      component[:settings][:'auto-scaling-group-name-output'] ||= []
+      outputs = component[:settings][:'auto-scaling-group-name-output'].map { |name| name.to_sym }
+      missing_output_keys = (outputs - component[:defined_outputs].keys)
+       @errors << "'#{missing_output_keys.map(&:to_s)}' is not a CF stack output" unless missing_output_keys.empty?
+    end
+
+    def check_cname_swap_options(component)
+      @errors << "dns-fqdn is required when using cname-swap deployment-strategy" unless component[:settings][:'dns-fqdn']
+      @errors << "dns-zone is required when using cname-swap deployment-strategy" unless component[:settings][:'dns-zone']
+      @errors << "'#{component[:settings][:'elb-name-output']}' is not a CF stack output, which is required by cname-swap deployment" unless component[:defined_outputs].keys.include?(component[:settings][:'elb-name-output'].to_sym)
+    end
+
+    def check_application_name
+      @config[:application] = "" unless @config[:application]
+      return @errors << "Application name is missing in config" unless @config[:application].length > 0
+      @errors << "Application name cannot be longer than 100 and can only contain letters, numbers, '-' and '.'" unless @config[:application] =~ /^[a-zA-Z0-9\.-]{1,100}$/
+    end
+
+    def check_components validate_inputs
+      @config[:components] ||= {}
+      return @errors << "At least one component must be defined in config" unless @config[:components].length > 0
+      deployable_components = @config[:targets] || []
+      component_targets = @config[:components].select { |key, value| deployable_components.include?(key.to_s) }
+      invalid_names = deployable_components - component_targets.keys.map(&:to_s)
+      @errors <<  "Found invalid deployment components #{invalid_names}" if invalid_names.size > 0
+      component_targets.each do |component_name, component|
+        component[:settings] ||= {}
+        component[:inputs] ||= {}
+        component[:defined_outputs] ||= {}
+        @errors << "Component name cannot be longer than 100 and can only contain letters, numbers, '-' and '.': #{component_name}" unless component_name =~ /^[A-Za-z0-9\.-]{1,100}$/
+        check_parameters component_name, component if validate_inputs
+        check_cname_swap_options(component) if component[:'deployment-strategy'] == 'cname-swap'
+        check_asg_name_output(component)
+        check_hooks(component)
+        check_component_options(component_name, component)
+      end
+    end
+
+    def check_component_options(name, component)
+      component.keys.each do |option|
+        @errors << "The option '#{option}' of the component '#{name}' is not valid" unless ComponentOptions.include?(option)
+      end
+    end
+
+    def check_hooks(component)
+      hook_names = [:'before-destroy', :'after-create', :'after-swap']
+      hook_names.each do |hook_name|
+        next unless component[hook_name] && component[hook_name].is_a?(Hash)
+        @errors << "Invalid hook '#{hook_name}'" unless component[hook_name][:file] || component[hook_name][:code]
+        check_hook_file(component, hook_name)
+      end
+    end
+
+    def check_hook_file(component, hook_name)
+      file_name = component[hook_name][:file]
+      return unless file_name
+      path = File.join(component[:config_dir], file_name)
+      @errors << "File '#{path}' does not exist, which is required by hook '#{hook_name}'" unless File.exists?(path)
+    end
+
+    def check_environments
+      @config[:environments] ||= {}
+      @config[:environments].each do | name, environment |
+        @errors << "Environment name cannot be longer than 12 and can only contain letters, numbers, '-' and '.': #{name}" unless name =~ /^[a-zA-Z0-9\.-]{1,12}$/
+        check_environment_options(name, environment)
+      end
+    end
+
+    def check_environment_options(name, environment)
+      environment.keys.each do |option|
+        @errors << "The option '#{option}' of the environment '#{name}' is not valid" unless EnvironmentOptions.include?(option)
+        end
+    end
+
+
+    def check_parameters(component_name, component)
+      component[:defined_parameters] ||= {}
+      component[:defined_outputs] ||= {}
+      component[:defined_parameters].each do | parameter_name, parameter |
+        if component[:inputs].keys.include?(parameter_name) || parameter[:Default]
+          check_output_reference(parameter_name, component_name)
+        else
+          @errors << "No input setting '#{parameter_name}' found for CF template parameter in component #{component_name}"
+        end
+      end
+      check_un_used_inputs(component_name, component)
+    end
+
+    def check_un_used_inputs(component_name, component)
+      component[:inputs].keys.each do |input|
+        unless component[:defined_parameters].keys.include?(input) || CommonInputs.include?(input)
+          message = "The input '#{input}' defined in the component '#{component_name}' is not used in the json template as a parameter"
+          if component[:settings][:'raise-error-for-unused-inputs']
+            @errors << message
+          else
+            @warnings << message
+          end
+        end
+      end
+    end
+
+
+    def check_output_reference(setting_name, component_name)
+      setting = @config[:components][component_name][:inputs][setting_name]
+      return unless setting.is_a?(Hash)
+      ref_component_name = setting[:component].to_sym
+      ref_component = @config[:components][ref_component_name]
+      output_key = setting[:'output-key'].to_sym
+      @errors << "No output '#{output_key}' found in CF template of component #{ref_component_name}, which is referenced by input setting '#{setting_name}' in component #{component_name}" unless ref_component[:defined_outputs].keys.include?(output_key)
+    end
+
+  end
+end

data/lib/cf_deployer/defaults.rb

@@ -0,0 +1,10 @@
+module CfDeployer
+  module Defaults
+    ELBName = 'ELBName'
+    AutoScalingGroupName = 'AutoScalingGroupName'
+    Timeout = 900
+    DNSDriver = 'CfDeployer::Driver::Route53'
+    RaiseErrorForUnusedInputs = false
+    KeepPreviousStack = true
+  end
+end