knife-cloudformation 0.1.22 → 0.2.0

data/lib/chef/knife/cloudformation_promote.rb

@@ -0,0 +1,40 @@
+require 'knife-cloudformation'
+
+class Chef
+  class Knife
+    class CloudformationPromote < Knife
+
+      include KnifeCloudformation::Knife::Base
+
+      banner 'knife cloudformation promote NEW_STACK_NAME DESTINATION'
+
+      option(:accounts,
+        :long => '--accounts-file PATH',
+        :short => '-A PATH',
+        :description => 'JSON account file',
+        :proc => lambda{|v|
+          Chef::Config[:knife][:cloudformation][:promote_accounts] = JSON.load(File.read(v))
+        }
+      )
+
+      option(:storage_bucket,
+        :long => '--exports-bucket NAME',
+        :description => 'Bucket name containing the exports',
+        :proc => lambda{|v| Chef::Config[:knife][:cloudformation][:promote_exports_bucket] = v }
+      )
+
+      option(:storage_prefix,
+        :long => '--exports-prefix PREFIX',
+        :description => 'Prefix of stack key',
+        :proc => lambda{|v| Chef::Config[:knife][:cloudformation][:promote_exports_prefix] = v }
+      )
+
+
+      def _run
+        stack_name, destination = name_args
+
+      end
+
+    end
+  end
+end

data/lib/chef/knife/cloudformation_update.rb

@@ -1,30 +1,147 @@
-require 'knife-cloudformation/cloudformation_base'
-require File.join(File.dirname(__FILE__), 'cloudformation_create')
+require 'knife-cloudformation'
 
 class Chef
   class Knife
     class CloudformationUpdate < Knife
-      banner 'knife cloudformation update NAME'
 
-      include KnifeCloudformation::KnifeBase
-      include CloudformationCreate::Options
+      include KnifeCloudformation::Knife::Base
+      include KnifeCloudformation::Knife::Template
+      include KnifeCloudformation::Knife::Stack
+
+      option(:file_path_prompt,
+        :long => '--[no-]file-path-prompt',
+        :description => 'Interactive prompt for template path discovery',
+        :boolean => true,
+        :default => false,
+        :proc => lambda {|val|
+          Chef::Config[:knife][:cloudformation][:file_path_prompt] = val
+        }
+      )
+      option(:apply_stacks,
+        :long => '--apply-stack NAME_OR_ID',
+        :description => 'Autofill parameters using existing stack outputs. Can be used multiple times',
+        :proc => lambda {|val|
+          Chef::Config[:knife][:cloudformation][:update] ||= Mash.new
+          Chef::Config[:knife][:cloudformation][:update][:apply_stacks] ||= []
+          Chef::Config[:knife][:cloudformation][:update][:apply_stacks].push(val).uniq!
+        }
+      )
 
-      def create_stack(name, stack)
-        begin
-          res = aws_con.update_stack(name, stack)
-        rescue => e
-          ui.fatal "Failed to update stack #{name}. Reason: #{e}"
-          _debug(e, "Generated template used:\n#{_format_json(stack['TemplateBody'])}")
+      banner 'knife cloudformation update NAME'
+
+      # Run the stack creation command
+      def _run
+        name = name_args.first
+        unless(name)
+          ui.fatal "Formation name must be specified!"
           exit 1
         end
+
+        stack = provider.stacks.get(name)
+
+        if(stack)
+          ui.info "#{ui.color('Cloud Formation:', :bold)} #{ui.color('update', :green)}"
+          file = load_template_file(:allow_missing)
+          stack_info = "#{ui.color('Name:', :bold)} #{name}"
+
+          if(Chef::Config[:knife][:cloudformation][:file])
+            stack_info << " #{ui.color('Path:', :bold)} #{Chef::Config[:knife][:cloudformation][:file]}"
+          else
+            stack_info << " #{ui.color('(no temlate update)', :yellow)}"
+          end
+          ui.info "  -> #{stack_info}"
+
+          apply_stacks!(stack)
+
+          if(file)
+            redefault_stack_parameters(file, stack)
+            populate_parameters!(file)
+            file = translate_template(file)
+            stack.template = file
+            stack.parameters = Chef::Config[:knife][:cloudformation][:parameters]
+          else
+            stack_parameters_update!(stack)
+          end
+
+          stack.save
+
+          if(Chef::Config[:knife][:cloudformation][:poll])
+            poll_stack(stack.name)
+            provider.fetch_stacks
+            if(stack.success?)
+              ui.info "Stack update complete: #{ui.color('SUCCESS', :green)}"
+              provider.fetch_stacks
+              knife_output = Chef::Knife::CloudformationDescribe.new
+              knife_output.name_args.push(name)
+              knife_output.config[:outputs] = true
+              knife_output.run
+            else
+              ui.fatal "Update of stack #{ui.color(name, :bold)}: #{ui.color('FAILED', :red, :bold)}"
+              ui.info ""
+              knife_inspect = Chef::Knife::CloudformationInspect.new
+              knife_inspect.name_args.push(name)
+              knife_inspect.config[:instance_failure] = true
+              knife_inspect.run
+              exit 1
+            end
+          else
+            ui.warn 'Stack state polling has been disabled.'
+            ui.info "Stack update initialized for #{ui.color(name, :green)}"
+          end
+        else
+          ui.fatal "Failed to locate requested stack: #{ui.color(name, :red, :bold)}"
+          exit -1
+        end
+      end
+
+      # Update default values for parameters in template with
+      # currently used parameters on the existing stack
+      #
+      # @param template [Hash] stack template
+      # @param stack [Fog::Orchestration::Stack]
+      # @return [Hash]
+      def redefault_stack_parameters(template, stack)
+        stack.parameters.each do |key, value|
+          if(template['Parameters'][key])
+            template['Parameters'][key]['Default'] = value
+          end
+        end
+        template
       end
 
-      def action_in_progress?(name)
-        stack_status(name) == 'UPDATE_IN_PROGRESS'
+      # Apply any defined remote stacks
+      #
+      # @param stack [Miasma::Models::Orchestration::Stack]
+      # @return [Miasma::Models::Orchestration::Stack]
+      def apply_stacks!(stack)
+        remote_stacks = Chef::Config[:knife][:cloudformation].
+          fetch(:update, {}).fetch(:apply_stacks, [])
+        remote_stacks.each do |stack_name|
+          remote_stack = provider.stacks.get(stack_name)
+          if(remote_stack)
+            remote_stack.parameters.each do |key, value|
+              next if Chef::Config[:knife][:cloudformation][:stacks][:ignore_parameters].include?(key)
+              if(stack.parameters.has_key?(key))
+                stack.parameters[key] = value
+              end
+            end
+          else
+            ui.error "Failed to apply requested stack. Unable to locate. (#{stack_name})"
+            exit 1
+          end
+        end
+        stack
       end
 
-      def action_successful?(name)
-        stack_status(name) == 'UPDATE_COMPLETE'
+      # Update parameters within existing stack
+      #
+      # @param stack [Miasma::Models::Orchestration::Stack]
+      # @return [Miasma::Models::Orchestration::Stack]
+      def stack_parameters_update!(stack)
+        stack.parameters.each do |key, value|
+          answer = ui.ask_question("#{key.split(/([A-Z]+[^A-Z]*)/).find_all{|s|!s.empty?}.join(' ')}: ", :default => value)
+          stack.parameters[key] = answer
+        end
       end
 
     end

data/lib/chef/knife/cloudformation_validate.rb

@@ -0,0 +1,35 @@
+require 'pathname'
+require 'sparkle_formation'
+require 'knife-cloudformation'
+
+class Chef
+  class Knife
+    # Cloudformation validate command
+    class CloudformationValidate < Knife
+
+      include KnifeCloudformation::Knife::Base
+      include KnifeCloudformation::Knife::Template
+
+      banner 'knife cloudformation validate'
+
+      def _run
+        file = load_template_file
+        ui.info "#{ui.color('Cloud Formation Validation: ', :bold)} #{Chef::Config[:knife][:cloudformation][:file].sub(Dir.pwd, '').sub(%r{^/}, '')}"
+        file = KnifeCloudformation::Utils::StackParameterScrubber.scrub!(file)
+        file = translate_template(file)
+        begin
+          result = provider.stacks.build(
+            :name => 'validation-stack',
+            :template => file
+          ).validate
+          ui.info ui.color('  -> VALID', :bold, :green)
+        rescue => e
+          ui.info ui.color('  -> INVALID', :bold, :red)
+          ui.fatal e.message
+          failed = true
+        end
+      end
+
+    end
+  end
+end

data/lib/knife-cloudformation.rb

@@ -1 +1,29 @@
 require 'knife-cloudformation/version'
+require 'miasma'
+
+module KnifeCloudformation
+
+  autoload :Provider, 'knife-cloudformation/provider'
+  autoload :Cache, 'knife-cloudformation/cache'
+  autoload :Export, 'knife-cloudformation/export'
+  autoload :Utils, 'knife-cloudformation/utils'
+  autoload :MonkeyPatch, 'knife-cloudformation/monkey_patch'
+  autoload :Knife, 'knife-cloudformation/knife'
+
+end
+
+class Chef
+  class Knife
+    autoload :CloudformationCreate, 'chef/knife/cloudformation_create'
+    autoload :CloudformationDescribe, 'chef/knife/cloudformation_describe'
+    autoload :CloudformationDestroy, 'chef/knife/cloudformation_destroy'
+    autoload :CloudformationEvents, 'chef/knife/cloudformation_events'
+    autoload :CloudformationExport, 'chef/knife/cloudformation_export'
+    autoload :CloudformationImport, 'chef/knife/cloudformation_import'
+    autoload :CloudformationInspect, 'chef/knife/cloudformation_inspect'
+    autoload :CloudformationList, 'chef/knife/cloudformation_list'
+    autoload :CloudformationPromote, 'chef/knife/cloudformation_promote'
+    autoload :CloudformationUpdate, 'chef/knife/cloudformation_update'
+    autoload :CloudformationValidate, 'chef/knife/cloudformation_validate'
+  end
+end

data/lib/knife-cloudformation/cache.rb

@@ -1,15 +1,29 @@
 require 'digest/sha2'
+require 'thread'
+require 'knife-cloudformation'
+
+begin
+  require 'redis-objects'
+rescue LoadError
+  $stderr.puts 'The `redis-objects` gem is required for Cache support!'
+  raise
+end
 
 module KnifeCloudformation
+  # Data caching helper
   class Cache
 
     class << self
 
+      # Configure the caching approach to use
+      #
+      # @param type [Symbol] :redis or :local
+      # @param args [Hash] redis connection arguments if used
       def configure(type, args={})
         type = type.to_sym
         case type
         when :redis
-          require 'redis-objects'
+          @_pid = Process.pid
           Redis::Objects.redis = Redis.new(args)
         when :local
         else
@@ -18,14 +32,24 @@ module KnifeCloudformation
         enable(type)
       end
 
+      # Set enabled caching type
+      #
+      # @param type [Symbol]
+      # @return [Symbol]
       def enable(type)
         @type = type.to_sym
       end
 
+      # @return [Symbol] type of caching enabled
       def type
         @type || :local
       end
 
+      # Set/get time limit on data type
+      #
+      # @param kind [String, Symbol] data type
+      # @param seconds [Integer]
+      # return [Integer] seconds
       def apply_limit(kind, seconds=nil)
         @apply_limit ||= {}
         if(seconds)
@@ -34,62 +58,109 @@ module KnifeCloudformation
         @apply_limit[kind.to_sym].to_i
       end
 
+      # @return [Hash] default limits
       def default_limits
         (@apply_limit || {}).dup
       end
 
+      # Ping the redis connection and reconnect if dead
+      def redis_ping!
+        if((@_pid && @_pid != Process.pid) || !Redis::Objects.redis.connected?)
+          Redis::Objects.redis.client.reconnect
+          @_pid = Process.pid
+        end
+      end
+
     end
 
+    # @return [String] custom key for this cache
     attr_reader :key
-    attr_reader :direct_store
 
+    # Create new instance
+    #
+    # @param key [String, Array]
     def initialize(key)
       if(key.respond_to?(:sort))
         key = key.flatten if key.respond_to?(:flatten)
         key = key.map(&:to_s).sort
       end
       @key = Digest::SHA256.hexdigest(key.to_s)
-      @direct_store = {}
       @apply_limit = self.class.default_limits
     end
 
+    # Initialize a new data type
+    #
+    # @param name [Symbol] name of data
+    # @param kind [Symbol] data type
+    # @param args [Hash] options for data type
     def init(name, kind, args={})
-      name = name.to_sym
-      unless(@direct_store[name])
-        full_name = [key, name.to_s].join('_')
-        @direct_store[name] = get_storage(self.class.type, kind, full_name, args)
-      end
+      get_storage(self.class.type, kind, name, args)
       true
     end
 
+    # @return [Hash] data registry
+    def registry
+      get_storage(self.class.type, :hash, "registry_#{key}")
+    end
+
+    # Clear data
+    #
+    # @param args [Symbol] list of names to delete
+    # @return [TrueClass]
+    # @note clears all data if no names provided
     def clear!(*args)
       internal_lock do
-        args = @direct_store.keys if args.empty?
+        args = registry.keys if args.empty?
         args.each do |key|
-          value = @direct_store[key]
+          value = self[key]
           if(value.respond_to?(:clear))
             value.clear
           elsif(value.respond_to?(:value))
             value.value = nil
           end
+          registry.delete(key)
         end
         yield if block_given?
       end
       true
     end
 
-    def get_storage(store_type, data_type, full_name, args={})
+    # Fetch item from storage
+    #
+    # @param store_type [Symbol]
+    # @param data_type [Symbol]
+    # @param name [Symbol] name of data
+    # @param args [Hash] options for underlying storage
+    # @return [Object]
+    def get_storage(store_type, data_type, name, args={})
+      full_name = "#{key}_#{name}"
+      result = nil
       case store_type.to_sym
       when :redis
-        get_redis_storage(data_type, full_name, args)
+        result = get_redis_storage(data_type, full_name.to_s, args)
       when :local
-        get_local_storage(data_type, full_name, args)
+        @_local_cache ||= {}
+        unless(@_local_cache[full_name.to_s])
+          @_local_cache[full_name.to_s] = get_local_storage(data_type, full_name.to_s, args)
+        end
+        result = @_local_cache[full_name.to_s]
       else
         raise TypeError.new("Unsupported caching storage type encountered: #{store_type}")
       end
+      unless(full_name == "#{key}_registry_#{key}")
+        registry[name.to_s] = data_type
+      end
+      result
     end
 
+    # Fetch item from redis storage
+    #
+    # @param data_type [Symbol]
+    # @param full_name [Symbol]
+    # @param args [Hash]
+    # @return [Object]
     def get_redis_storage(data_type, full_name, args={})
+      self.class.redis_ping!
       case data_type.to_sym
       when :array
         Redis::List.new(full_name, {:marshal => true}.merge(args))
@@ -98,7 +169,7 @@ module KnifeCloudformation
       when :value
         Redis::Value.new(full_name, {:marshal => true}.merge(args))
       when :lock
-        Redis::Lock.new(full_name, {:expiration => 3, :timeout => 0.1}.merge(args))
+        Redis::Lock.new(full_name, {:expiration => 60, :timeout => 0.1}.merge(args))
       when :stamped
         Stamped.new(full_name.sub("#{key}_", '').to_sym, get_redis_storage(:value, full_name), self)
       else
@@ -106,43 +177,76 @@ module KnifeCloudformation
       end
     end
 
+    # Fetch item from local storage
+    #
+    # @param data_type [Symbol]
+    # @param full_name [Symbol]
+    # @param args [Hash]
+    # @return [Object]
+    # @todo make proper singleton for local storage
     def get_local_storage(data_type, full_name, args={})
-      case data_type.to_sym
-      when :array
-        []
-      when :hash
-        {}
-      when :value
-        LocalValue.new
-      when :lock
-        LocalLock.new
-      when :stamped
-        Stamped.new(full_name.sub("#{key}_", '').to_sym, get_local_storage(:value, full_name), self)
-      else
-        raise TypeError.new("Unsupported caching data type encountered: #{data_type}")
-      end
+      @storage ||= {}
+      @storage[full_name] ||= case data_type.to_sym
+        when :array
+          []
+        when :hash
+          {}
+        when :value
+          LocalValue.new
+        when :lock
+          LocalLock.new(full_name, {:expiration => 60, :timeout => 0.1}.merge(args))
+        when :stamped
+          Stamped.new(full_name.sub("#{key}_", '').to_sym, get_local_storage(:value, full_name), self)
+        else
+          raise TypeError.new("Unsupported caching data type encountered: #{data_type}")
+        end
     end
 
+    # Execute block within internal lock
+    #
+    # @return [Object] result of yield
+    # @note for internal use
     def internal_lock
-      get_storage(self.class.type, :lock, :internal_access, :timeout => 20).lock do
+      get_storage(self.class.type, :lock, :internal_access, :timeout => 20, :expiration => 120).lock do
         yield
       end
     end
 
+    # Fetch data
+    #
+    # @param name [String, Symbol]
+    # @return [Object, NilClass]
     def [](name)
-      internal_lock do
-        @direct_store[name.to_sym]
+      if(kind = registry[name.to_s])
+        get_storage(self.class.type, kind, name)
+      else
+        nil
       end
     end
 
+    # Set data
+    #
+    # @param key [Object]
+    # @param val [Object]
+    # @note this will never work, thus you should never use it
     def []=(key, val)
       raise 'Setting backend data is not allowed'
     end
 
+    # Check if cache time has expired
+    #
+    # @param key [String, Symbol] value key
+    # @param stamp [Time, Integer]
+    # @return [TrueClass, FalseClass]
     def time_check_allow?(key, stamp)
       Time.now.to_i - stamp.to_i > apply_limit(key)
     end
 
+    # Apply time limit for data type
+    #
+    # @param kind [String, Symbol] data type
+    # @param seconds [Integer]
+    # return [Integer]
     def apply_limit(kind, seconds=nil)
       @apply_limit ||= {}
       if(seconds)
@@ -151,50 +255,124 @@ module KnifeCloudformation
       @apply_limit[kind.to_sym].to_i
     end
 
+    # Perform action within lock
+    #
+    # @param lock_name [String, Symbol] name of lock
+    # @param raise_on_locked [TrueClass, FalseClass] raise execption if lock wait times out
+    # @return [Object] result of yield
+    def locked_action(lock_name, raise_on_locked=false)
+      begin
+        self[lock_name].lock do
+          yield
+        end
+      rescue Redis::Lock::LockTimeout
+        raise if raise_on_locked
+      end
+    end
 
+    # Simple value for memory cache
     class LocalValue
+      # @return [Object] value
       attr_accessor :value
       def initialize(*args)
         @value = nil
       end
     end
 
+    # Simple lock for memory cache
     class LocalLock
-      def initialize(*args)
+
+      # @return [Symbol] key name
+      attr_reader :_key
+      # @return [Numeric] timeout
+      attr_reader :_timeout
+      # @return [Mutex] underlying lock
+      attr_reader :_lock
+
+      # Create new instance
+      #
+      # @param name [Symbol] name of lock
+      # @param args [Hash]
+      # @option args [Numeric] :timeout
+      def initialize(name, args={})
+        @_key = name
+        @_timeout = args.fetch(:timeout, -1).to_f
+        @_lock = Mutex.new
       end
 
+      # Aquire lock and yield
+      #
+      # @yield block to execute within lock
+      # @return [Object] result of yield
       def lock
-        yield
+        locked = false
+        attempt_start = Time.now.to_f
+        while(!locked && (_timeout < 0 || Time.now.to_f - attempt_start < _timeout))
+          locked = _lock.try_lock
+        end
+        if(locked)
+          begin
+            yield
+          ensure
+            _lock.unlock if _lock.locked?
+          end
+        else
+          raise Redis::Lock::LockTimeout.new "Timeout on lock #{_key} exceeded #{_timeout} sec"
+        end
       end
 
+      # Clear the lock
+      #
+      # @note this is a noop
       def clear
+        # noop
       end
     end
 
+    # Wrapper to auto stamp values
     class Stamped
 
+      # Create new instance
+      #
+      # @param name [String, Symbol]
+      # @param base [Redis::Value, LocalValue]
+      # @param cache [Cache]
       def initialize(name, base, cache)
         @name = name.to_sym
         @base = base
         @cache = cache
       end
 
+      # @return [Object] value stored
       def value
         @base.value[:value] if set?
       end
 
+      # Store value and update timestamp
+      #
+      # @param v [Object] value
+      # @return [Object]
       def value=(v)
-        @base.value = {:stamp => Time.now.to_i, :value => v}
+        @base.value = {:stamp => Time.now.to_f, :value => v}
+        v
       end
 
+      # @return [TrueClass, FalseClass] is value set
       def set?
         @base.value.is_a?(Hash)
       end
 
+      # @return [Float] timestamp of last set (or 0.0 if unset)
       def stamp
-        @base.value[:stamp] if set?
+        set? ? @base.value[:stamp] : 0.0
+      end
+
+      # Force a timestamp update
+      def restamp!
+        self.value = value
       end
 
+      # @return [TrueClass, FalseClass] update is allowed based on stamp and limits
       def update_allowed?
         !set? || @cache.time_check_allow?(@name, @base.value[:stamp])
       end