cuffsert 0.9.0

data/lib/cuffsert/main.rb

@@ -0,0 +1,129 @@
+require 'cuffsert/cfarguments'
+require 'cuffsert/cfstates'
+require 'cuffsert/cli_args'
+require 'cuffsert/confirmation'
+require 'cuffsert/messages'
+require 'cuffsert/metadata'
+require 'cuffsert/presenters'
+require 'cuffsert/rxcfclient'
+require 'rx'
+require 'uri'
+
+# TODO:
+# - Stop using file: that we anyway need to special-case in cfarguments
+# - default value for meta.metadata when stack_path is local file
+# - selector and metadata are mandatory and need guards accordingly
+# - validate_and_urlify belongs in metadata.rb
+# - execute should use helpers and not know details of statuses
+# - update 'abort' should delete cheangeset and emit the result
+
+module CuffSert
+  def self.validate_and_urlify(stack_path)
+    if stack_path =~ /^[A-Za-z0-9]+:/
+      stack_uri = URI.parse(stack_path)
+    else
+      normalized = File.expand_path(stack_path)
+      unless File.exist?(normalized)
+        raise "Local file #{normalized} does not exist"
+      end
+      stack_uri = URI.join('file:///', normalized)
+    end
+    unless ['s3', 'file'].include?(stack_uri.scheme)
+      raise "Uri #{stack_uri.scheme} is not supported"
+    end
+    stack_uri
+  end
+
+  def self.create_stack(client, meta, confirm_create)
+    cfargs = CuffSert.as_create_stack_args(meta)
+    Rx::Observable.concat(
+      Rx::Observable.of([:create, meta.stackname]),
+      Rx::Observable.defer do
+        if confirm_create.call(meta, :create, nil)
+          client.create_stack(cfargs)
+        else
+          Abort.new('User abort!').as_observable
+        end
+      end
+    )
+  end
+
+  def self.update_stack(client, meta, confirm_update)
+    cfargs = CuffSert.as_update_change_set(meta)
+    client.prepare_update(cfargs)
+      .last
+      .flat_map do |change_set|
+        Rx::Observable.concat(
+          Rx::Observable.of(change_set),
+          Rx::Observable.defer {
+            if change_set[:status] == 'FAILED'
+              client.abort_update(change_set[:change_set_id])
+            elsif confirm_update.call(meta, :update, change_set)
+              client.update_stack(change_set[:stack_id], change_set[:change_set_id])
+            else
+              Rx::Observable.concat(
+                client.abort_update(change_set[:change_set_id]),
+                Abort.new('User abort!').as_observable
+              )
+            end
+          }
+        )
+      end
+  end
+
+  def self.recreate_stack(client, stack, meta, confirm_recreate)
+    crt_args = CuffSert.as_create_stack_args(meta)
+    del_args = CuffSert.as_delete_stack_args(stack)
+    Rx::Observable.concat(
+      Rx::Observable.of([:recreate, stack]),
+      Rx::Observable.defer do
+        if confirm_recreate.call(meta, :recreate, stack)
+          Rx::Observable.concat(
+            client.delete_stack(del_args),
+            client.create_stack(crt_args)
+          )
+        else
+          CuffSert::Abort.new('User abort!').as_observable
+        end
+      end
+    )
+  end
+
+  def self.execute(meta, confirm_update, force_replace: false, client: RxCFClient.new)
+    sources = []
+    found = client.find_stack_blocking(meta)
+
+    if found && INPROGRESS_STATES.include?(found[:stack_status])
+      sources << Abort.new('Stack operation already in progress').as_observable
+    elsif found.nil?
+      sources << self.create_stack(client, meta, confirm_update)
+    elsif found[:stack_status] == 'ROLLBACK_COMPLETE' || force_replace
+      sources << self.recreate_stack(client, found, meta, confirm_update)
+    else
+      sources << self.update_stack(client, meta, confirm_update)
+    end
+    Rx::Observable.concat(*sources)
+  end
+
+  def self.make_renderer(cli_args)
+    if cli_args[:output] == :json
+      JsonRenderer.new(STDOUT, STDERR, cli_args)
+    else
+      ProgressbarRenderer.new(STDOUT, STDERR, cli_args)
+    end
+  end
+  
+  def self.run(argv)
+    cli_args = CuffSert.parse_cli_args(argv)
+    meta = CuffSert.build_meta(cli_args)
+    if cli_args[:stack_path].nil? || cli_args[:stack_path].size != 1
+      raise 'Requires exactly one stack path'
+    end
+    stack_path = cli_args[:stack_path][0]
+    meta.stack_uri = CuffSert.validate_and_urlify(stack_path)
+    events = CuffSert.execute(meta, CuffSert.method(:confirmation),
+      force_replace: cli_args[:force_replace])
+    renderer = CuffSert.make_renderer(cli_args)
+    RendererPresenter.new(events, renderer)
+  end
+end

data/lib/cuffsert/messages.rb

@@ -0,0 +1,18 @@
+module CuffSert
+  class Abort
+    attr_reader :message
+
+    def initialize(message)
+      @message = message
+    end
+
+    def ===(other)
+      # For the benefit of value_matches? and regex
+      other.is_a?(Abort) && (other.message === @message || @message === other.message)
+    end
+
+    def as_observable
+      Rx::Observable.just(self)
+    end
+  end
+end

data/lib/cuffsert/metadata.rb

@@ -0,0 +1,90 @@
+require 'yaml'
+
+module CuffSert
+  class StackConfig
+    attr_accessor :stackname, :selected_path, :op_mode, :stack_uri
+    attr_accessor :suffix, :parameters, :tags
+
+    def initialize
+      @selected_path = []
+      @op_mode = :normal
+      @parameters = {}
+      @tags = {}
+    end
+
+    def append_path(lmnt)
+      @selected_path << lmnt
+    end
+
+    def update_from(metadata)
+      @stackname = metadata[:stackname] || @stackname
+      @suffix = metadata[:suffix] || @suffix
+      @parameters.merge!(metadata[:parameters] || {})
+      @tags.merge!(metadata[:tags] || {})
+      self
+    end
+
+    def stackname
+      @stackname || (@selected_path + [*@suffix]).join('-')
+    end
+  end
+
+  def self.load_config(io)
+    config = YAML.load(io)
+    raise 'config does not seem to be a YAML hash?' unless config.is_a?(Hash)
+    config = symbolize_keys(config)
+    format = config.delete(:format)
+    raise 'Please include Format: v1' if format.nil? || format.downcase != 'v1'
+    config
+  end
+
+  def self.meta_for_path(metadata, path, target = StackConfig.new)
+    target.update_from(metadata)
+    candidate, path = path
+    key = candidate || metadata[:defaultpath]
+    variants = metadata[:variants]
+    if key.nil?
+      raise "No DefaultPath found for #{variants.keys}" unless variants.nil?
+      return target
+    end
+    target.append_path(key)
+
+    raise "Missing variants section as expected by #{key}" if variants.nil?
+    new_meta = variants[key.to_sym]
+    raise "#{key.inspect} not found in variants" if new_meta.nil?
+    self.meta_for_path(new_meta, path, target)
+  end
+
+  def self.build_meta(cli_args)
+    io = open(cli_args[:metadata])
+    config = CuffSert.load_config(io)
+    default = self.meta_defaults(cli_args)
+    meta = CuffSert.meta_for_path(config, cli_args[:selector], default)
+    CuffSert.cli_overrides(meta, cli_args)
+  end
+
+  private_class_method
+
+  def self.meta_defaults(cli_args)
+    default = StackConfig.new
+    default.suffix = File.basename(cli_args[:metadata], '.yml')
+    default
+  end
+
+  def self.cli_overrides(meta, cli_args)
+    meta.update_from(cli_args[:overrides])
+    meta.op_mode = cli_args[:op_mode] || meta.op_mode
+    meta
+  end
+
+  def self.symbolize_keys(hash)
+    hash.each_with_object({}) do |(k, v), h|
+      k = k.downcase.to_sym
+      if k == :tags || k == :parameters
+        h[k] = v.each_with_object({}) { |e, h| h[e['Name']] = e['Value'] }
+      else
+        h[k] = v.is_a?(Hash) ? symbolize_keys(v) : v
+      end
+    end
+  end
+end

data/lib/cuffsert/presenters.rb

@@ -0,0 +1,313 @@
+require 'aws-sdk'
+require 'colorize'
+require 'cuffsert/cfstates'
+require 'cuffsert/messages'
+require 'rx'
+
+# TODO: Animate in-progress states
+# - Introduce a Done message and stop printing in on_complete
+# - Present the error message in change_set properly - and abort
+# - badness goes to stderr
+# - change sets should present modification details indented under each entry
+#   - property direct modification
+#   - properties through parameter change
+#   - indirect change through other resource ("causing_entity": "Lb.DNSName")
+
+module CuffSert
+  class BasePresenter
+    def initialize(events)
+      events.subscribe(
+        method(:on_event),
+        method(:on_error),
+        method(:on_complete)
+      )
+    end
+
+    def on_error(err)
+      STDERR.puts'Error:'
+      STDERR.puts err
+      STDERR.puts err.backtrace.join("\n\t")
+    end
+
+    def on_complete
+    end
+
+    def update_width(width)
+    end
+  end
+
+  class RawPresenter < BasePresenter
+    def on_event(event)
+      puts event.inspect
+    end
+
+    def on_complete
+      puts 'Done.'
+    end
+  end
+
+  class RendererPresenter < BasePresenter
+    def initialize(events, renderer)
+      @resources = []
+      @index = {}
+      @renderer = renderer
+      super(events)
+    end
+
+    def on_event(event)
+      # Workaround for now
+      event = event.data if event.class == Seahorse::Client::Response
+
+      case event
+      when Aws::CloudFormation::Types::StackEvent
+        on_stack_event(event)
+      when Aws::CloudFormation::Types::DescribeChangeSetOutput
+        on_change_set(event)
+      # when [:recreate, Aws::CloudFormation::Types::Stack]
+      when Array
+        on_stack(*event)
+      when ::CuffSert::Abort
+        @renderer.abort(event)
+      else
+        puts event
+      end
+    end
+
+    def on_complete
+      @renderer.done
+    end
+
+    private
+
+    def on_change_set(change_set)
+      @renderer.change_set(change_set.to_h)
+    end
+
+    def on_stack_event(event)
+      resource = lookup_stack_resource(event)
+      update_resource_states(resource, event)
+      @renderer.event(event, resource)
+      @renderer.clear
+      @resources.each { |resource| @renderer.resource(resource) }
+      clear_resources if is_completed_stack_event(event)
+    end
+
+    def on_stack(event, stack)
+      @renderer.stack(event, stack)
+    end
+
+    def lookup_stack_resource(event)
+      rid = event[:logical_resource_id]
+      unless (pos = @index[rid])
+        pos = @index[rid] = @resources.size
+        @resources << make_resource(event)
+      end
+      @resources[pos]
+    end
+
+    def make_resource(event)
+      event.to_h
+        .reject { |k, _| k == :timestamp }
+        .merge!(:states => [])
+    end
+
+    def update_resource_states(resource, event)
+      resource[:states] = resource[:states]
+        .reject { |state| state == :progress }
+        .take(1) << CuffSert.state_category(event[:resource_status])
+    end
+
+    def is_completed_stack_event(event)
+      event[:resource_type] == 'AWS::CloudFormation::Stack' &&
+        FINAL_STATES.include?(event[:resource_status])
+    end
+
+    def clear_resources
+      @resources.clear
+      @index.clear
+    end
+  end
+
+  class BaseRenderer
+    def initialize(output = STDOUT, error = STDERR, options = {})
+      @output = output
+      @error = error
+      @verbosity = options[:verbosity] || 1
+    end
+
+    def change_set(change_set) ; end
+    def event(event, resource) ; end
+    def clear ; end
+    def resource(resource) ; end
+    def abort(message) ; end
+    def done ; end
+  end
+
+  class JsonRenderer < BaseRenderer
+    def change_set(change_set)
+      @output.write(change_set.to_h.to_json) unless @verbosity < 1
+    end
+
+    def event(event, resource)
+      @output.write(event.to_h.to_json) unless @verbosity < 1
+    end
+
+    def stack(event, stack)
+      @output.write(stack.to_json) unless @verbosity < 1
+    end
+
+    def abort(event)
+      @error.write(event.message + "\n") unless @verbosity < 1
+    end
+  end
+
+  ACTION_ORDER = ['Add', 'Modify', 'Replace?', 'Replace!', 'Remove']
+
+  class ProgressbarRenderer < BaseRenderer
+    def change_set(change_set)
+      @output.write(sprintf("Updating stack %s\n", change_set[:stack_name]))
+      change_set[:changes].sort do |l, r|
+        lr = l[:resource_change]
+        rr = r[:resource_change]
+        [
+          ACTION_ORDER.index(action(lr)),
+          lr[:logical_resource_id]
+        ] <=> [
+          ACTION_ORDER.index(action(rr)),
+          rr[:logical_resource_id]
+        ]
+      end.map do |change|
+        rc = change[:resource_change]
+        sprintf("%s[%s] %-10s %s\n",
+          rc[:logical_resource_id],
+          rc[:resource_type],
+          action_color(action(rc)),
+          scope_desc(rc)
+        )
+      end.each { |row| @output.write(row) }
+    end
+
+    def action(rc)
+      if rc[:action] == 'Modify'
+        if ['True', 'Always'].include?(rc[:replacement])
+          'Replace!'
+        elsif ['False', 'Never'].include?(rc[:replacement])
+          'Modify'
+        elsif rc[:replacement] == 'Conditional'
+          'Replace?'
+        else
+          "#{rc[:action]}/#{rc[:replacement]}"
+        end
+      else
+        rc[:action]
+      end
+    end
+
+    def action_color(action)
+      action.colorize(
+        case action
+        when 'Add' then :green
+        when 'Modify' then :yellow
+        else :red
+        end
+      )
+    end
+
+    def scope_desc(rc)
+      (rc[:scope] || []).map do |scope|
+        case scope
+        when 'Properties'
+          properties = rc[:details]
+            .select { |detail| detail[:target][:attribute] == 'Properties' }
+            .map { |detail| detail[:target][:name] }
+            .uniq
+            .join(", ")
+          sprintf("Properties: %s", properties)
+        else
+          rc[:scope]
+        end
+      end
+      .join("; ")
+    end
+
+    def event(event, resource)
+      return if @verbosity == 0
+      return if resource[:states][-1] != :bad && @verbosity <= 1
+      color, _ = interpret_states(resource)
+      message = sprintf('%s %s  %s[%s] %s',
+        event[:resource_status],
+        event[:timestamp].strftime('%H:%M:%S%z'),
+        event[:logical_resource_id],
+        event[:resource_type].sub(/.*::/, ''),
+        event[:resource_status_reason] || ""
+      ).colorize(color)
+      @output.write("\r#{message}\n")
+    end
+
+    def stack(event, stack)
+      case event
+      when :create
+        @output.write("Creating stack #{stack}\n")
+      when :recreate
+        message = sprintf(
+          "Deleting and re-creating stack %s",
+          stack[:stack_name]
+        )
+        @output.write(message.colorize(:red) + "\n")
+      else
+        puts event, stack
+      end
+    end
+
+    def clear
+      @output.write("\r") unless @verbosity < 1
+    end
+
+    def resource(resource)
+      return if @verbosity < 1
+      color, symbol = interpret_states(resource)
+      table = {
+        :check => "+",
+        :tripple_dot => ".", # "\u2026"
+        :cross  => "!",
+        :qmark => "?",
+      }
+
+      @output.write(table[symbol].colorize(
+        :color => :white,
+        :background => color
+      ))
+    end
+
+    def abort(event)
+      @error.write(event.message.colorize(:red) + "\n") unless @verbosity < 1
+    end
+
+    def done
+      @output.write("\nDone.\n".colorize(:green)) unless @verbosity < 1
+    end
+
+    private
+
+    def interpret_states(resource)
+      case resource[:states]
+      when [:progress]
+        [:yellow, :tripple_dot]
+      when [:good]
+        [:green, :check]
+      when [:bad]
+        [:red, :cross]
+      when [:good, :progress]
+        [:light_white, :tripple_dot]
+      when [:bad, :progress]
+        [:red, :tripple_dot]
+      when [:good, :good], [:bad, :good]
+        [:light_white, :check]
+      when [:good, :bad], [:bad, :bad]
+        [:red, :qmark]
+      else
+        raise "Unexpected :states in #{resource.inspect}"
+      end
+    end
+  end
+end

data/lib/cuffsert/rxcfclient.rb

@@ -0,0 +1,116 @@
+require 'aws-sdk'
+require 'cuffsert/cfstates'
+require 'rx'
+
+module CuffSert
+  class RxCFClient
+    def initialize(
+        aws_cf = Aws::CloudFormation::Client.new(retry_limit: 8),
+        pause: 5,
+        max_items: 1000)
+      @cf = aws_cf
+      @max_items = max_items
+      @pause = pause
+    end
+
+    def find_stack_blocking(meta)
+      name = meta.stackname
+      @cf.describe_stacks(stack_name: name)[:stacks][0]
+    rescue Aws::CloudFormation::Errors::ValidationError
+      nil
+    end
+
+    def create_stack(cfargs)
+      Rx::Observable.create do |observer|
+        start_time = record_start_time
+        stack_id = @cf.create_stack(cfargs)[:stack_id]
+        stack_events(stack_id, start_time) do |event|
+          observer.on_next(event)
+        end
+        observer.on_completed
+      end
+    end
+
+    def prepare_update(cfargs)
+      Rx::Observable.create do |observer|
+        change_set_id = @cf.create_change_set(cfargs)[:id]
+        loop do
+          change_set = @cf.describe_change_set(change_set_name: change_set_id)
+          observer.on_next(change_set)
+          break if FINAL_STATES.include?(change_set[:status])
+        end
+        observer.on_completed
+      end
+    end
+
+    def update_stack(stack_id, change_set_id)
+      Rx::Observable.create do |observer|
+        start_time = record_start_time
+        @cf.execute_change_set(change_set_name: change_set_id)
+        stack_events(stack_id, start_time) do |event|
+          observer.on_next(event)
+        end
+        observer.on_completed
+      end
+    end
+    
+    def abort_update(change_set_id)
+      Rx::Observable.create do |observer|
+        @cf.delete_change_set(change_set_name: change_set_id)
+        observer.on_completed
+      end
+    end
+
+    def delete_stack(cfargs)
+      eventid_cache = Set.new
+      Rx::Observable.create do |observer|
+        start_time = record_start_time
+        @cf.delete_stack(cfargs)
+        stack_events(cfargs[:stack_name], start_time) do |event|
+          observer.on_next(event)
+        end
+        observer.on_completed
+      end
+      .select do |event|
+        eventid_cache.add?(event[:event_id])
+      end
+    end
+
+    private
+
+    def record_start_time
+      # Please make sure your machine has NTP :p
+      DateTime.now - 5.0 / 86400
+    end
+
+    def stack_finished?(stack_id, event)
+      event[:physical_resource_id] == stack_id &&
+        FINAL_STATES.include?(event[:resource_status])
+    end
+
+    def flatten_events(stack_id)
+      @cf.describe_stack_events(stack_name: stack_id).each do |events|
+        for event in events[:stack_events]
+          yield event
+        end
+      end
+    end
+
+    def stack_events(stack_id, start_time)
+      eventid_cache = Set.new
+      loop do
+        events = []
+        done = false
+        flatten_events(stack_id) do |event|
+          break if event[:timestamp].to_datetime < start_time
+          next unless eventid_cache.add?(event[:event_id])
+          events.unshift(event)
+          done = true if stack_finished?(stack_id, event)
+        end
+        events.each { |event| yield event }
+        break if done
+        sleep(@pause)
+      end
+    end
+  end
+end