cfn-flow 0.2.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e16917c5b5720382b9419f1dd7e4be47c0f0cbf2
-  data.tar.gz: 1448b8d55c7679a1a20c2f622b58ba9397ee9f69
+  metadata.gz: a8a7c12f98182c07a7961ddca970637d6454275a
+  data.tar.gz: 4af32ae281b4a00f55eb9a7263d56eb40fefb678
 SHA512:
-  metadata.gz: 20c957cf75facb44eb17cd36795858e891cc7b23ddbc900e4ee752fcd6561f0be31cc0f2df7f9d11b432ee00191607dbdef8e11b42fde07be6d4f2bbae6ddac4
-  data.tar.gz: 5e2876dbca28e35787d383d6d1f0858608c167bb2e4b389c362d000c1f6930af7b10efc94b7db755cbf0dc2ae85d71c82096c9c34f5b49444c66b35d467a17b4
+  metadata.gz: 012e7d1fd2c3d517ef83725ef4a29ed4165669cf235a744be0481ac09cb18479b5a5e77b350b76e810b6e2eb52daa2264c0999f784bd57dcb0a2d2b59d1fb0a6
+  data.tar.gz: 1f6bfd0cb3466afa28c054dcd7c03a3dae04bc729578fb475af346f18ecb699dbf258ec306caf8c657088b09c48a3db1dcc02edd400f910d896c8a19a1749106

data/README.md

@@ -1,5 +1,10 @@
 # cfn-flow
-An opinionated command-line workflow for developing AWS CloudFormation templates. Track template changes in git and upload versioned releases to AWS S3.
+An opinionated command-line workflow for developing [AWS CloudFormation](https://aws.amazon.com/cloudformation/) templates and deploying stacks.
+
+Track template changes in git and upload versioned releases to AWS S3.
+
+Deploy stacks using a standard, reliable process with extensible
+configuration in git.
 
 #### Opinions
 

data/Rakefile

@@ -4,6 +4,6 @@ require "bundler/gem_tasks"
 require 'rake/testtask'
 
 Rake::TestTask.new do |t|
-  t.pattern = "spec/*_spec.rb"
+  t.pattern = "spec/**/*_spec.rb"
 end
 task :default => :test

data/lib/cfn-flow.rb

@@ -2,8 +2,138 @@ require 'thor'
 require 'aws-sdk'
 require 'multi_json'
 require 'yaml'
+require 'erb'
+
+module CfnFlow
+  class << self
+
+    ##
+    # Configuration
+    def config_path
+      ENV['CFN_FLOW_CONFIG_PATH'] || 'cfn-flow.yml'
+    end
+
+    def load_config
+      @config = YAML.load(
+        ERB.new( File.read(config_path) ).result(binding)
+      )
+      # TODO: Validate config?
+    end
+
+    def config_loaded?
+      @config.is_a? Hash
+    end
+
+    def config
+      load_config unless config_loaded?
+      @config
+    end
+
+    def service
+      unless config.key?('service')
+        raise Thor::Error.new("No service name in #{config_path}. Add 'service: my_app_name'.")
+      end
+      config['service']
+    end
+
+    def stack_params(environment)
+      unless config['stack'].is_a? Hash
+        raise Thor::Error.new("No stack defined in #{config_path}. Add 'stack: ...'.")
+      end
+
+      # Dup & symbolize keys
+      params = config['stack'].map{|k,v| [k.to_sym, v]}.to_h
+
+      # Expand params
+      if params[:parameters].is_a? Hash
+        expanded_params = params[:parameters].map do |key,value|
+          { parameter_key: key, parameter_value: value }
+        end
+        params[:parameters] = expanded_params
+      end
+
+      # Expand tags
+      if params[:tags].is_a? Hash
+        tags = params[:tags].map do |key, value|
+          {key: key, value: value}
+        end
+
+        params[:tags] = tags
+      end
+
+      # Append CfnFlow tags
+      params[:tags] ||= []
+      params[:tags] << { key: 'CfnFlowService', value: service }
+      params[:tags] << { key: 'CfnFlowEnvironment', value: environment }
+
+      # Expand template body
+      if params[:template_body].is_a? String
+        begin
+          body = CfnFlow::Template.new(params[:template_body]).to_json
+          params[:template_body] = body
+        rescue CfnFlow::Template::Error
+          # Do nothing
+        end
+      end
+
+      params
+    end
+
+    def template_s3_bucket
+      unless config['templates'].is_a?(Hash) &&  config['templates']['s3_bucket']
+        raise Thor::Error.new("No s3_bucket defined for templates in #{config_path}. Add 'templates: { s3_bucket: ... }'.")
+      end
+
+      config['templates']['s3_bucket']
+    end
+
+    def template_s3_prefix
+      unless config['templates'].is_a?(Hash)
+        raise Thor::Error.new("No templates defined in #{config_path}. Add 'templates: ... '.")
+      end
+
+      # Ok for this to be ''
+      config['templates']['s3_prefix']
+    end
+
+    ##
+    # Aws Clients
+    def cfn_client
+      @cfn_client ||= Aws::CloudFormation::Client.new(region: config[:region] || ENV['AWS_REGION'])
+    end
+
+    def cfn_resource
+      # NB: increase default retry limit to avoid throttling errors iterating over stacks.
+      # See https://github.com/aws/aws-sdk-ruby/issues/705
+      @cfn_resource ||= Aws::CloudFormation::Resource.new(
+        region: config[:region] || ENV['AWS_REGION'],
+        retry_limit: 10
+      )
+    end
+
+    # Clear aws sdk clients & config (for tests)
+    def clear!
+      @config = @cfn_client = @cfn_resource = nil
+    end
+
+    # Exit with status code = 1 when raising a Thor::Error
+    # Override thor default
+    def exit_on_failure?
+      if instance_variable_defined?(:@exit_on_failure)
+       @exit_on_failure
+      else
+        true
+      end
+    end
+
+    def exit_on_failure=(value)
+      @exit_on_failure = value
+    end
+  end
+end
 
-module CfnFlow; end
 require 'cfn-flow/template'
 require 'cfn-flow/git'
+require 'cfn-flow/event_presenter'
 require 'cfn-flow/cli'
+require 'cfn-flow/version'

data/lib/cfn-flow/cli.rb

@@ -1,119 +1,175 @@
-class CfnFlow::CLI < Thor
-
-  def self.shared_options
-    method_option :bucket,     type: :string, desc: 'S3 bucket for templates'
-    method_option :to,         type: :string, desc: 'S3 path prefix for templates'
-    method_option :from,       type: :string, desc: 'Local source directory for templates'
-    method_option 'dev-name',  type: :string, desc: 'Personal development prefix'
-    method_option :region,     type: :string, desc: 'AWS Region'
-    method_option :verbose,    type: :boolean, desc: 'Verbose output', default: false
-  end
+module CfnFlow
+  class CLI < Thor
 
-  # Exit with status code = 1 when raising a Thor::Error
-  # Override thor default
-  def self.exit_on_failure?
-    true
-  end
+    def self.exit_on_failure?
+      CfnFlow.exit_on_failure?
+    end
 
-  no_commands do
-    def load_config
-      defaults = { 'from' => '.' }
-      file_config = begin
-                      YAML.load_file(ENV['CFN_FLOW_CONFIG'] || './cfn-flow.yml')
-                    rescue Errno::ENOENT
-                      {}
-                    end
-      env_config = {
-        'bucket'      => ENV['CFN_FLOW_BUCKET'],
-        'to'          => ENV['CFN_FLOW_TO'],
-        'from'        => ENV['CFN_FLOW_FROM'],
-        'dev-name'    => ENV['CFN_FLOW_DEV_NAME'],
-        'region'      => ENV['AWS_REGION']
-      }.delete_if {|_,v| v.nil?}
-
-      # Env vars override config file. Command args override env vars.
-      self.options = defaults.merge(file_config).merge(env_config).merge(options)
-
-      # Ensure region env var is set for AWS client
-      ENV['AWS_REGION'] = options['region']
-
-      # validate required options are present
-      %w(region bucket to from).each do |arg|
-        unless options[arg]
-          raise Thor::RequiredArgumentMissingError.new("Missing required argument '#{arg}'")
-        end
+    ##
+    # Template methods
+
+    desc 'validate TEMPLATE [...]', 'Validates templates'
+    def validate(*templates)
+
+      if templates.empty?
+        raise Thor::RequiredArgumentMissingError.new('You must specify a template to validate')
       end
 
-      unless options['dev-name'] || options['release']
-        raise Thor::RequiredArgumentMissingError.new("Missing either 'dev-name' or 'release' argument")
+      templates.map{|path| Template.new(path) }.each do |template|
+        say "Validating #{template.local_path}... "
+        template.validate!
+        say 'valid.', :green
       end
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      raise Thor::Error.new("Invalid template. Message: #{e.message}")
+    rescue CfnFlow::Template::Error => e
+      raise Thor::Error.new("Error loading template. (#{e.class}) Message: #{e.message}")
     end
 
-    shared_options
-    def load_templates
-      load_config
-      glob = File.join(options['from'], '**/*.{yml,json,template}')
-
-      puts Dir.glob(glob.inspect)
-      @templates = Dir.glob(glob).map { |path|
-        CfnFlow::Template.new(from: path, bucket: options['bucket'], prefix: prefix)
-      }.select {|t|
-        verbose "Checking file #{t.from}... "
-        if t.is_cfn_template?
-          verbose "loaded"
-          true
-        else
-          verbose "skipped."
-          false
-        end
-      }
+    desc 'publish TEMPLATE [...]', 'Validate & upload templates'
+    method_option 'dev-name', type: :string, desc: 'Personal development prefix'
+    method_option :release,   type: :string, desc: 'Upload release', lazy_default: CfnFlow::Git.sha
+    method_option :verbose,   type: :boolean, desc: 'Verbose output', default: false
+    def publish(*templates)
+      if templates.empty?
+        raise Thor::RequiredArgumentMissingError.new('You must specify a template to publish')
+      end
+
+      validate(*templates)
+      # TODO: check git is clean before releasing
+      #CfnFlow::Git.check_status if options['release']
+
+      release = publish_release
+      templates.each do |path|
+        t = Template.new(path)
+
+        say "Publishing #{t.local_path} to #{t.url(release)}"
+        t.upload(release)
+      end
     end
-  end
 
-  desc :validate, 'Validates templates'
-  shared_options
-  def validate
-    load_templates
-    @templates.each do |t|
+    ##
+    # Stack methods
+
+    desc 'deploy ENVIRONMENT', 'Launch a stack'
+    method_option :cleanup, type: :boolean, desc: 'Prompt to shutdown other stacks in ENVIRONMENT after launching'
+    def deploy(environment)
+
       begin
-        say "Validating #{t.from}... "
-        t.validate!
-        say "valid."
-      rescue Aws::CloudFormation::Errors::ValidationError
-        raise Thor::Error.new("Error validating #{t.from}. Message: #{$!.message}")
+        params = CfnFlow.stack_params(environment)
+        stack = CfnFlow.cfn_resource.create_stack(params)
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        raise Thor::Error.new(e.message)
+      end
+
+      say "Launching stack #{stack.name}"
+
+      # Invoke events
+      say "Polling for events..."
+      invoke :events, [stack.name], ['--poll']
+
+
+      # Optionally cleanup other stacks in this environment
+      if options[:cleanup]
+        puts "Finding stacks to clean up"
+        list_stacks_in_service.select {|s|
+          s.name != stack.name && \
+            s.tags.any? {|tag| tag.key == 'CfnFlowEnvironment' && tag.value == environment }
+        }.map(&:name).each do |name|
+          delete(name)
+        end
       end
     end
-  end
 
-  desc :upload, 'Validate & upload templates to the CFN_FLOW_DEV_NAME prefix'
-  shared_options
-  method_option :release, type: :string, lazy_default: CfnFlow::Git.sha, desc: 'Upload release'
-  def upload
-    CfnFlow::Git.check_status if options['release']
+    desc 'list [ENVIRONMENT]', 'List running stacks in all environments, or ENVIRONMENT'
+    method_option 'no-header', type: :boolean, desc: 'Do not print column headers'
+    def list(environment=nil)
+      stacks = list_stacks_in_service
+      if environment
+        stacks.select! do |stack|
+          stack.tags.any? {|tag| tag.key == 'CfnFlowEnvironment' && tag.value == environment }
+        end
+      end
+
+      return if stacks.empty?
+
+      table_header = options['no-header'] ? [] : [['NAME',  'ENVIRONMENT', 'STATUS']]
+      table_data = stacks.map do |s|
+        env_tag = s.tags.detect {|tag| tag.key == 'CfnFlowEnvironment'}
+        env = env_tag ? env_tag.value : 'NONE'
 
-    validate
-    @templates.each do |t|
-      say "Uploading #{t.from} to #{t.url}"
-      t.upload!
+        [ s.name, env, s.stack_status ]
+      end
+
+      print_table(table_header + table_data)
     end
 
-  end
-  default_task :upload
+    desc 'show STACK', 'Show details about STACK'
+    method_option :json, type: :boolean, desc: 'Show stack as JSON (default is YAML)'
+    def show(name)
+      data = find_stack_in_service(name).data.to_hash
+      say options[:json] ? MultiJson.dump(data, pretty: true) : data.to_yaml
+    end
 
-  private
-  def verbose(msg)
-    say msg if options['verbose']
-  end
+    desc 'events STACK', 'List events for  STACK'
+    method_option :poll, type: :boolean, desc: 'Poll for new events until the stack is complete'
+    method_option 'no-header', type: :boolean, desc: 'Do not print column headers'
+    def events(name)
+      stack = find_stack_in_service(name)
+
+      say EventPresenter.header unless options['no-header']
+      EventPresenter.present(stack.events) {|p| say p }
 
-  def prefix
-    # Add the release or dev name to the prefix
-    parts = []
-    parts << options['to']
-    if options['release']
-      parts += [ 'release',  options['release'] ]
-    else
-      parts += [ 'dev', options['dev-name'] ]
+      if options[:poll]
+        # Display events until we're COMPLETE/FAILED
+        delay = (ENV['CFN_FLOW_EVENT_POLL_INTERVAL'] || 2).to_i
+        stack.wait_until(max_attempts: -1, delay: delay) do |s|
+          EventPresenter.present(s.events) {|p| say p }
+          # Wait until the stack status ends with _FAILED or _COMPLETE
+          s.stack_status.match(/_(FAILED|COMPLETE)$/)
+        end
+      end
+    end
+
+    desc 'delete STACK', 'Shut down STACK'
+    method_option :force, type: :boolean, default: false, desc: 'Shut down without confirmation'
+    def delete(name)
+      stack = find_stack_in_service(name)
+      if options[:force] || yes?("Are you sure you want to shut down #{name}?", :red)
+        stack.delete
+        say "Deleted stack #{name}"
+      end
+    end
+
+    private
+    def find_stack_in_service(name)
+      stack = CfnFlow.cfn_resource.stack(name).load
+      unless stack.tags.any? {|tag| tag.key == 'CfnFlowService' && tag.value == CfnFlow.service }
+        raise Thor::Error.new "Stack #{name} is not tagged for service #{CfnFlow.service}"
+      end
+      stack
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      # Handle missing stacks: 'Stack with id blah does not exist'
+      raise Thor::Error.new(e.message)
+    end
+
+    def list_stacks_in_service
+      CfnFlow.cfn_resource.stacks.select do |stack|
+        stack.tags.any? {|tag| tag.key == 'CfnFlowService' && tag.value == CfnFlow.service }
+      end
+    end
+
+    def publish_release
+      # Add the release or dev name to the prefix
+      if options[:release]
+        'release/' + options[:release]
+      elsif options['dev-name']
+        'dev/' + options['dev-name']
+      elsif ENV['CFN_FLOW_DEV_NAME']
+        'dev/' + ENV['CFN_FLOW_DEV_NAME']
+      else
+        raise Thor::Error.new("Must specify --release or --dev-name; or set CFN_FLOW_DEV_NAME env var")
+      end
     end
-    File.join(*parts)
   end
 end

data/lib/cfn-flow/event_presenter.rb

@@ -0,0 +1,39 @@
+require 'set'
+module CfnFlow
+  class EventPresenter
+
+    ##
+    # Class methods
+    def self.seen_event_ids
+      @seen_event_ids ||= Set.new
+    end
+
+    # Yields each new event present to +block+
+    def self.present(raw_events, &block)
+      raw_events.to_a.reverse.sort_by(&:timestamp).
+        reject {|e| seen_event_ids.include?(e.id) }.
+        map    {|e| yield new(e) }
+    end
+
+    def self.header
+      %w(status logical_resource_id resource_type reason) * "\t"
+    end
+
+    ##
+    # Instance methods
+    attr_accessor :event
+    def initialize(event)
+      @event = event
+      self.class.seen_event_ids << event.id
+    end
+
+    def to_s
+        [
+          event.resource_status,
+          event.logical_resource_id,
+          event.resource_type,
+          event.resource_status_reason
+        ].compact * "\t"
+    end
+  end
+end