knife-cloudformation 0.1.12 → 0.1.14

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## v0.1.14
+* Extract template building tools
+* Add support for custom CF locations and prompting
+* Updates in fetching and caching behavior
+
 ## v0.1.12
 * Use the split value when re-joining parameters
 

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/knife-cloudformation.gemspec

@@ -10,8 +10,10 @@ Gem::Specification.new do |s|
   s.description = 'Knife tooling for Cloud Formation'
   s.require_path = 'lib'
   s.add_dependency 'chef'
-  s.add_dependency 'fog', '~> 1.15'
+  s.add_dependency 'fog', '~> 1.17'
   s.add_dependency 'net-sftp'
-  s.add_dependency 'attribute_struct', '~> 0.1.6'
+  s.add_dependency 'sparkle_formation', '~> 0.1.2'
+  s.add_dependency 'redis-objects'
+  s.add_dependency 'attribute_struct', '~> 0.1.8'
   s.files = Dir['**/*']
 end

data/lib/chef/knife/cloudformation_create.rb

@@ -1,4 +1,5 @@
-require 'knife-cloudformation/sparkle_formation'
+require 'sparkle_formation'
+require 'pathname'
 require 'chef/knife/cloudformation_base'
 
 class Chef
@@ -94,6 +95,16 @@ class Chef
                 :long => '--print-only',
                 :description => 'Print template and exit'
               )
+              option(:base_directory,
+                :long => '--cloudformation-directory PATH',
+                :description => 'Path to cloudformation directory',
+                :proc => lambda {|val| Chef::Config[:knife][:cloudformation][:base_directory] = val}
+              )
+              option(:no_base_directory,
+                :long => '--no-cloudformation-directory',
+                :description => 'Unset any value used for cloudformation path',
+                :proc => lambda {|*val| Chef::Config[:knife][:cloudformation][:base_directory] = nil}
+              )
 
               %w(rollback polling interactive_parameters).each do |key|
                 if(Chef::Config[:knife][:cloudformation][key].nil?)
@@ -109,13 +120,20 @@ class Chef
 
       def run
         @action_type = self.class.name.split('::').last.sub('Cloudformation', '').upcase
+        name = name_args.first
+        unless(name)
+          ui.fatal "Formation name must be specified!"
+          exit 1
+        end
+
+        set_paths_and_discover_file!
         unless(File.exists?(Chef::Config[:knife][:cloudformation][:file].to_s))
           ui.fatal "Invalid formation file path provided: #{Chef::Config[:knife][:cloudformation][:file]}"
           exit 1
         end
-        name = name_args.first
+
         if(Chef::Config[:knife][:cloudformation][:processing])
-          file = KnifeCloudformation::SparkleFormation.compile(Chef::Config[:knife][:cloudformation][:file])
+          file = SparkleFormation.compile(Chef::Config[:knife][:cloudformation][:file])
         else
           file = _from_json(File.read(Chef::Config[:knife][:cloudformation][:file]))
         end
@@ -176,6 +194,94 @@ class Chef
         end
       end
 
+      private
+
+      def set_paths_and_discover_file!
+        if(Chef::Config[:knife][:cloudformation][:base_directory])
+          SparkleFormation.components_path = File.join(
+            Chef::Config[:knife][:cloudformation][:base_directory], 'components'
+          )
+          SparkleFormation.dynamics_path = File.join(
+            Chef::Config[:knife][:cloudformation][:base_directory], 'dynamics'
+          )
+        end
+        unless(Chef::Config[:knife][:cloudformation][:file])
+          Chef::Config[:knife][:cloudformation][:file] = prompt_for_file(
+            Chef::Config[:knife][:cloudformation][:base_directory] || File.join(Dir.pwd, 'cloudformation')
+          )
+        else
+          unless(Pathname(Chef::Config[:knife][:cloudformation][:file]).absolute?)
+            Chef::Config[:knife][:cloudformation][:file] = File.join(
+              Chef::Config[:knife][:cloudformation][:base_directory] || File.join(Dir.pwd, 'cloudformation'),
+              Chef::Config[:knife][:cloudformation][:file]
+            )
+          end
+        end
+      end
+
+      IGNORE_CF_DIRS = %w(dynamics components)
+
+      def prompt_for_file(dir)
+        directory = Dir.new(dir)
+        directories = directory.map do |d|
+          if(!d.start_with?('.') && !IGNORE_CF_DIRS.include?(d) && File.directory?(path = File.join(dir, d)))
+            path
+          end
+        end.compact.sort
+        files = directory.map do |f|
+          if(!f.start_with?('.') && File.file?(path = File.join(dir, f)))
+            path
+          end
+        end.compact.sort
+        if(directories.empty? && files.empty?)
+          ui.fatal 'No formation paths discoverable!'
+        else
+          output = ['Please select the formation to create']
+          output << '(or directory to list):' unless directories.empty?
+          ui.info output.join(' ')
+          output.clear
+          idx = 1
+          valid = {}
+          unless(directories.empty?)
+            output << ui.color('Directories:', :bold)
+            directories.each do |path|
+              valid[idx] = {:path => path, :type => :directory}
+              output << [idx, "#{File.basename(path).sub('.rb', '').split(/[-_]/).map(&:capitalize).join(' ')}"]
+              idx += 1
+            end
+          end
+          unless(files.empty?)
+            output << ui.color('Templates:', :bold)
+            files.each do |path|
+              valid[idx] = {:path => path, :type => :file}
+              output << [idx, "#{File.basename(path).sub('.rb', '').split(/[-_]/).map(&:capitalize).join(' ')}"]
+              idx += 1
+            end
+          end
+          max = idx.to_s.length
+          output.map! do |o|
+            if(o.is_a?(Array))
+              "  #{o.first}.#{' ' * (max - o.first.to_s.length)} #{o.last}"
+            else
+              o
+            end
+          end
+          ui.info "#{output.join("\n")}\n"
+          response = ask_question('Enter selection: ').to_i
+          unless(valid[response])
+            ui.fatal 'How about using a real value'
+            exit 1
+          else
+            entry = valid[response.to_i]
+            if(entry[:type] == :directory)
+              prompt_for_file(entry[:path])
+            else
+              Chef::Config[:knife][:cloudformation][:file] = entry[:path]
+            end
+          end
+        end
+      end
+
     end
   end
 end

data/lib/chef/knife/cloudformation_destroy.rb

@@ -26,8 +26,12 @@ class Chef
           ui.info "Destroy request sent for stack: #{ui.color(stack_name, :bold)}"
         end
         if(config[:polling])
-          stacks.each do |stack_name|
-            poll_stack(stack_name)
+          begin
+            stacks.each do |stack_name|
+              poll_stack(stack_name)
+            end
+          rescue Fog::AWS::CloudFormation::NotFound
+            # ignore this error since this is the end result we want!
           end
           ui.info "  -> Destroyed Cloud Formation#{plural}: #{ui.color(stacks.join(', '), :bold, :red)}"
         end

data/lib/chef/knife/cloudformation_list.rb

@@ -38,10 +38,35 @@ class Chef
 
       def get_list
         get_things do
-          aws.stacks
+          aws.aws(:cloud_formation).list_stacks(list_options).body['StackSummaries'].sort do |x,y|
+            if(y['CreationTime'].to_s.empty?)
+              -1
+            elsif(x['CreationTime'].to_s.empty?)
+              1
+            else
+              Time.parse(y['CreationTime'].to_s) <=> Time.parse(x['CreationTime'].to_s)
+            end
+          end
         end
       end
 
+      def list_options
+        status = Chef::Config[:knife][:cloudformation][:status] ||
+          KnifeCloudformation::AwsCommons::DEFAULT_STACK_STATUS
+        if(status.map(&:downcase).include?('none'))
+          filter = {}
+        else
+          count = 0
+          filter = Hash[*(
+              status.map do |n|
+                count += 1
+                ["StackStatusFilter.member.#{count}", n]
+              end.flatten
+          )]
+        end
+        filter
+      end
+
       def default_attributes
         %w(StackName CreationTime StackStatus TemplateDescription)
       end

data/lib/knife-cloudformation/aws_commons/stack.rb

@@ -1,9 +1,12 @@
-require 'knife-cloudformation/aws_commons.rb'
+require 'knife-cloudformation/cache'
+require 'knife-cloudformation/aws_commons'
+require 'digest/sha2'
 
 module KnifeCloudformation
   class AwsCommons
     class Stack
 
+      include KnifeCloudformation::Utils::Debug
       include KnifeCloudformation::Utils::JSON
       include KnifeCloudformation::Utils::AnimalStrings
 
@@ -27,7 +30,7 @@ module KnifeCloudformation
         def build_stack_definition(template, options={})
           stack = Mash.new
           options.each do |key, value|
-            format_key = key.split('_').map do |k|
+            format_key = key.to_s.split('_').map do |k|
               "#{k.slice(0,1).upcase}#{k.slice(1,k.length)}"
             end.join
             stack[format_key] = value
@@ -61,12 +64,19 @@ module KnifeCloudformation
       def initialize(name, common, raw_stack=nil)
         @name = name
         @common = common
-        @memo = {}
+        @memo = Cache.new(common.credentials.merge(:stack => name))
+        reset_local
+        @memo.init(:raw_stack, :stamped)
         if(raw_stack)
-          @raw_stack = raw_stack
-        else
-          load_stack
+          if(@memo[:stacks])
+            if(@memo[:stacks].stamp > @memo[:raw_stack].stamp)
+              @memo[:raw_stack].value = raw_stack
+            end
+          else
+            @memo[:raw_stack].value = raw_stack
+          end
         end
+        load_stack
         @force_refresh = false
         @force_refresh = in_progress?
       end
@@ -99,27 +109,63 @@ module KnifeCloudformation
         res
       end
 
-      def load_stack
-        @raw_stack = common.aws(:cloud_formation)
-          .describe_stacks('StackName' => name)
-          .body['Stacks'].first
+      def load_stack(*args)
+        @memo.init(:raw_stack, :stamped)
+        begin
+          @memo.init(:raw_stack_lock, :lock)
+          @memo[:raw_stack_lock].lock do
+            if(args.include?(:force) || @memo[:raw_stack].update_allowed?)
+              @memo[:raw_stack].value = common.aws(:cloud_formation)
+                .describe_stacks('StackName' => name)
+                .body['Stacks'].first
+            end
+          end
+        rescue => e
+          if(defined?(Redis) && e.is_a?(Redis::Lock::LockTimeout))
+            # someone else is updating
+            debug 'Got lock timeout on stack load'
+          else
+            raise
+          end
+        end
       end
 
       def load_resources
-        @raw_resources = common.aws(:cloud_formation)
-          .describe_stack_resources('StackName' => name)
-          .body['StackResources']
+        @memo.init(:raw_resources, :stamped)
+        begin
+          @memo.init(:raw_resources_lock, :lock)
+          @memo[:raw_resources_lock].lock do
+            if(@memo[:raw_resources].update_allowed?)
+              @memo[:raw_resources].value = common.aws(:cloud_formation)
+                .describe_stack_resources('StackName' => name)
+                .body['StackResources']
+            end
+          end
+        rescue => e
+          if(defined?(Redis) && e.is_a?(Redis::Lock::LockTimeout))
+            debug 'Got lock timeout on resource load'
+          else
+            raise e
+          end
+        end
       end
 
       def refresh?(bool=nil)
         bool || (bool.nil? && @force_refresh)
       end
 
+      def reset_local
+        @local = {
+          :nodes => []
+        }
+      end
+
       def reload!
-        load_stack
-        load_resources
-        @force_refresh = in_progress?
-        @memo = {}
+        @memo.clear! do
+          load_stack(:force)
+          load_resources
+          @force_refresh = in_progress?
+        end
         true
       end
 
@@ -141,99 +187,142 @@ module KnifeCloudformation
       end
 
       def template
-        unless(@memo[:template])
-          @memo[:template] = _from_json(
+        @memo.init(:template, :value)
+        unless(@memo[:template].value)
+          @memo[:template].value = _from_json(
             common.aws(:cloud_formation)
               .get_template(name).body['TemplateBody']
           )
         end
-        @memo[:template]
+        @memo[:template].value
       end
 
       ## Stack metadata ##
       def parameters(raw=false)
         if(raw)
-          @raw_stack['Parameters']
+          @memo[:raw_stack].value['Parameters']
         else
-          unless(@memo[:parameters])
-            @memo[:parameters] = Hash[*(
-                @raw_stack['Parameters'].map do |ary|
+          @memo.init(:parameters, :value)
+          unless(@memo[:parameters].value)
+            @memo[:parameters].value = Hash[*(
+                @memo[:raw_stack].value['Parameters'].map do |ary|
                   [ary['ParameterKey'], ary['ParameterValue']]
                 end.flatten
             )]
           end
-          @memo[:parameters]
+          @memo[:parameters].value
         end
       end
 
       def capabilities
-        @raw_stack['Capabilities']
+        @memo[:raw_stack].value['Capabilities']
       end
 
       def disable_rollback
-        @raw_stack['DisableRollback']
+        @memo[:raw_stack].value['DisableRollback']
       end
 
       def notification_arns
-        @raw_stack['NotificationARNs']
+        @memo[:raw_stack].value['NotificationARNs']
       end
 
       def timeout_in_minutes
-        @raw_stack['TimeoutInMinutes']
+        @memo[:raw_stack].value['TimeoutInMinutes']
       end
       alias_method :timeout_in_minutes, :timeout
 
       def stack_id
-        @raw_stack['StackId']
+        @memo[:raw_stack].value['StackId']
       end
       alias_method :id, :stack_id
 
       def creation_time
-        @raw_stack['CreationTime']
+        @memo[:raw_stack].value['CreationTime']
       end
       alias_method :created_at, :creation_time
 
       def status(force_refresh=nil)
         load_stack if refresh?(force_refresh)
-        @raw_stack['StackStatus']
+        @memo[:raw_stack].value['StackStatus']
       end
 
       def resources(force_refresh=nil)
-        load_resources if @raw_resources.nil? || refresh?(force_refresh)
-        @raw_resources
+        load_resources if @memo[:raw_resources].nil? || refresh?(force_refresh)
+        @memo[:raw_resources].value
       end
 
       def events(all=false)
-        if(@memo[:events].nil? || refresh?)
-          res = common.aws(:cloud_formation).describe_stack_events(name).body['StackEvents']
-          @memo[:events] ||= []
-          current = @memo[:events].map{|e| e['EventId']}
-          res.delete_if{|e| current.include?(e['EventId'])}
-          @memo[:events] += res
-          @memo[:events].uniq!
-          @memo[:events].sort!{|x,y| x['Timestamp'] <=> y['Timestamp']}
-        else
-          res = []
+        @memo.init(:events, :stamped)
+        res = []
+        if(@memo[:events].value.nil? || refresh?)
+          begin
+            @memo.init(:events_lock, :lock)
+            @memo[:events_lock].lock do
+              if(@memo[:events].update_allowed?)
+                res = common.aws(:cloud_formation).describe_stack_events(name).body['StackEvents']
+                current = @memo[:events].value || []
+                current_events = current.map{|e| e['EventId']}
+                res.delete_if{|e| current_events.include?(e['EventId'])}
+                current += res
+                current.uniq!
+                current.sort!{|x,y| x['Timestamp'] <=> y['Timestamp']}
+                @memo[:events].value = current
+              end
+            end
+          rescue => e
+            if(defined?(Redis) && e.is_a?(Redis::Lock::LockTimeout))
+              debug 'Got lock timeout on events'
+            else
+              raise
+            end
+          end
         end
-        all ? @memo[:events] : res
+        all ? @memo[:events].value : res
       end
 
       def outputs(style=:unformatted)
         case style
         when :formatted
           Hash[*(
-              @raw_stack['Outputs'].map do |item|
+              @memo[:raw_stack].value['Outputs'].map do |item|
                 [item['OutputKey'].gsub(/(?<![A-Z])([A-Z])/, '_\1').sub(/^_/, '').downcase.to_sym, item['OutputValue']]
               end.flatten
           )]
         when :unformatted
           Hash[*(
-              @raw_stack['Outputs'].map do |item|
+              @memo[:raw_stack].value['Outputs'].map do |item|
                 [item['OutputKey'], item['OutputValue']]
               end.flatten
           )]
         else
-          @raw_stack['Outputs']
+          @memo[:raw_stack].value['Outputs']
+        end
+      end
+
+      def event_start_index(given_events, status)
+        Array(given_events).flatten.compact.rindex do |e|
+          e['ResourceType'] == 'AWS::CloudFormation::Stack' &&
+            e['ResourceStatus'] == status.to_s.upcase
+        end.to_i
+      end
+
+      # min:: do not return value lower than this (defaults to 5)
+      # Returns Numeric < 100 to represent completed resources
+      # percentage (never returns less than 5)
+      def percent_complete(min=5)
+        if(complete?)
+          100
+        else
+          all_events = events(:all)
+          total_expected = template['Resources'].size
+          action = performing
+          start = event_start_index(all_events, "#{action}_in_progress".to_sym)
+          finished = all_events.find_all do |e|
+            e['ResourceStatus'] == "#{action}_complete".upcase ||
+            e['ResourceStatus'] == "#{action}_failed".upcase
+          end.size
+          calculated = ((finished / total_expected.to_f) * 100).to_i
+          calculated < min ? min : calculated
         end
       end
 
@@ -254,7 +343,42 @@ module KnifeCloudformation
       end
 
       def success?
-        !failed?
+        !failed? && complete?
+      end
+
+      def creating?
+        in_progress? && status.to_s.downcase.start_with?('create')
+      end
+
+      def deleting?
+        in_progress? && status.to_s.downcase.start_with?('delete')
+      end
+
+      def updating?
+        in_progress? && status.to_s.downcase.start_with?('update')
+      end
+
+      def rollbacking?
+        in_progress? && status.to_s.downcase.start_with?('rollback')
+      end
+
+      def performing
+        if(in_progress?)
+          status.to_s.downcase.split('_').first.to_sym
+        end
+      end
+
+      # Lets build in some color coding!
+      def red?
+        failed? || deleting?
+      end
+
+      def yellow?
+        !red? && !green?
+      end
+
+      def green?
+        success? || creating? || updating?
       end
 
       ## Fog instance helpers ##
@@ -271,18 +395,34 @@ module KnifeCloudformation
       end
 
       def nodes
-        reload! if refresh?
-        unless(@memo[:nodes])
-          as_resources = resources.find_all{|r|r['ResourceType'] == 'AWS::AutoScaling::AutoScalingGroup'}
-          @memo[:nodes] =
-            as_resources.map do |as_resource|
+        if(@local[:nodes].empty?)
+          as_resources = resources.find_all do |r|
+            r['ResourceType'] == 'AWS::AutoScaling::AutoScalingGroup'
+          end
+          @local[:nodes] = as_resources.map do |as_resource|
             as_group = expand_resource(as_resource)
             as_group.instances.map do |inst|
               common.aws(:ec2).servers.get(inst.id)
             end
           end.flatten
         end
-        @memo[:nodes]
+        @local[:nodes]
+      end
+
+      def nodes_data(*args)
+        cache_key = ['nd', name, Digest::SHA256.hexdigest(args.map(&:to_s).join)].join('_')
+        @memo.init(cache_key, :value)
+        unless(@memo[cache_key].value)
+          data = nodes.map do |n|
+            [:id, args].flatten.compact.map do |k|
+              n.send(k)
+            end
+          end
+        end
+        unless(data.empty?)
+          @memo[cache_key].value = data
+        end
+        @memo[cache_key].value || data
       end
 
     end

data/lib/knife-cloudformation/aws_commons.rb

@@ -1,5 +1,6 @@
 require 'fog'
 require 'knife-cloudformation/utils'
+require 'knife-cloudformation/cache'
 
 Dir.glob(File.join(File.dirname(__FILE__), 'aws_commons/*.rb')).each do |item|
   require "knife-cloudformation/aws_commons/#{File.basename(item).sub('.rb', '')}"
@@ -8,26 +9,29 @@ end
 module KnifeCloudformation
   class AwsCommons
 
+    include KnifeCloudformation::Utils::AnimalStrings
+    include KnifeCloudformation::Utils::Debug
+
     FOG_MAP = {
       :ec2 => :compute
     }
 
+    attr_reader :credentials
+
     def initialize(args={})
       @ui = args[:ui]
-      @creds = args[:fog]
+      @credentials = @creds = args[:fog]
       @connections = {}
-      @memo = {
-        :stacks => {},
-        :event_ids => [],
-        :stack_list => {}
-      }
+      @memo = Cache.new(credentials)
+      @local = {:stacks => {}}
+    end
+
+    def cache
+      @memo
     end
 
     def clear_cache(*types)
-      keys = types.empty? ? @memo.keys : types.map(&:to_sym)
-      keys.each do |key|
-        @memo[key].clear if @memo[key]
-      end
+      @memo.clear!(*types)
       true
     end
 
@@ -43,9 +47,20 @@ module KnifeCloudformation
           dns_creds.delete(:region) || dns_creds.delete('region')
           @connections[:dns] = Fog::DNS::AWS.new(dns_creds)
         else
-          Fog.credentials = Fog.symbolize_credentials(@creds)
-          @connections[type] = Fog::AWS[type]
-          Fog.credentials = {}
+          begin
+            Fog.credentials = Fog.symbolize_credentials(@creds)
+            @connections[type] = Fog::AWS[type]
+            Fog.credentials = {}
+          rescue NameError
+            klass = [camel(type.to_s), 'AWS'].inject(Fog) do |memo, item|
+              memo.const_defined?(item) ? memo.const_get(item) : break
+            end
+            if(klass)
+              @connections[type] = klass.new(Fog.symbolize_credentials(@creds))
+            else
+              raise
+            end
+          end
         end
       end
       @connections[type]
@@ -62,64 +77,55 @@ module KnifeCloudformation
     )
 
     def stacks(args={})
-      status = args[:status] || DEFAULT_STACK_STATUS
-      key = status.hash
-      @memo[:stack_list].delete(key) if args[:force_refresh]
-      count = 0
-      if(status.map(&:downcase).include?('none'))
-        filter = {}
+      status = Array(args[:status] || DEFAULT_STACK_STATUS).flatten.compact.map do |stat|
+        stat.to_s.upcase
+      end
+      @memo.init(:stacks_lock, :lock)
+      @memo.init(:stacks, :stamped)
+      if(args[:cache_time])
+        @memo[:stacks].stamp
       else
-        filter = Hash[*(
-            status.map do |n|
-              count += 1
-              ["StackStatusFilter.member.#{count}", n]
-            end.flatten
-        )]
+        if(args[:refresh_every])
+          cache.apply_limit(:stacks, args[:refresh_every].to_i)
+        end
+        if(@memo[:stacks].update_allowed? || args[:force_refresh])
+          @memo[:stacks_lock].lock do
+            @memo[:stacks].value = aws(:cloud_formation).describe_stacks.body['Stacks']
+          end
+        end
       end
-      unless(@memo[:stack_list][key])
-        @memo[:stack_list][key] = aws(:cloud_formation).list_stacks(filter).body['StackSummaries']
+      @memo[:stacks].value.find_all do |s|
+        status.include?(s['StackStatus'])
       end
-      @memo[:stack_list][key]
     end
 
-    def name_from_stack_id(name)
+    def name_from_stack_id(s_id)
       found = stacks.detect do |s|
-        s['StackId'] == name
+        s['StackId'] == s_id
       end
-      if(found)
-        s['StackName']
-      else
-        raise "Failed to locate stack with ID: #{name}"
+      found ? found['StackName'] : raise(IndexError.new("Failed to locate stack with ID: #{s_id}"))
+    end
+
+    def id_from_stack_name(name)
+      found = stacks.detect do |s|
+        s['StackName'] == name
       end
+      found ? found['StackId'] : raise(IndexError.new("Failed to locate stack with name: #{name}"))
     end
 
     def stack(*names)
-      names = names.map do |name|
-        if(name.start_with?('arn:'))
-          name_from_stack_id(name)
-        else
-          name
-        end
-      end
-      if(names.size == 1)
-        name = names.first
-        unless(@memo[:stacks][name])
-          @memo[:stacks][name] = Stack.new(name, self)
-        end
-        @memo[:stacks][name]
-      else
-        to_fetch = names - @memo[:stacks].keys
-        slim_stacks = {}
-        unless(to_fetch.empty?)
-          to_fetch.each do |name|
-            slim_stacks[name] = Stack.new(name, self, stacks.detect{|s| s['StackName'] == name})
+      result = names.map do |name|
+        [name, name.start_with?('arn:') ? name : id_from_stack_name(name)]
+      end.map do |name, s_id|
+        unless(@local[:stacks][s_id])
+          seed = stacks.detect do |stk|
+            stk['StackId'] == s_id
           end
+          @local[:stacks][s_id] = Stack.new(name, self, seed)
         end
-        result = names.map do |n|
-          @memo[:stacks][n] || slim_stacks[n]
-        end
-        result
+        @local[:stacks][s_id]
       end
+      result.size == 1 ? result.first : result
     end
 
     def create_stack(name, definition)
@@ -131,7 +137,7 @@ module KnifeCloudformation
     def process(things, args={})
       @event_ids ||= []
       processed = things.reverse.map do |thing|
-        next if @memo[:event_ids].include?(thing['EventId'])
+        next if @event_ids.include?(thing['EventId'])
         @event_ids.push(thing['EventId']).compact!
         if(args[:attributes])
           args[:attributes].map do |key|

data/lib/knife-cloudformation/cache.rb

@@ -0,0 +1,204 @@
+require 'digest/sha2'
+
+module KnifeCloudformation
+  class Cache
+
+    class << self
+
+      def configure(type, args={})
+        type = type.to_sym
+        case type
+        when :redis
+          require 'redis-objects'
+          Redis::Objects.redis = Redis.new(args)
+        when :local
+        else
+          raise TypeError.new("Unsupported caching type: #{type}")
+        end
+        enable(type)
+      end
+
+      def enable(type)
+        @type = type.to_sym
+      end
+
+      def type
+        @type || :local
+      end
+
+      def apply_limit(kind, seconds=nil)
+        @apply_limit ||= {}
+        if(seconds)
+          @apply_limit[kind.to_sym] = seconds.to_i
+        end
+        @apply_limit[kind.to_sym].to_i
+      end
+
+      def default_limits
+        (@apply_limit || {}).dup
+      end
+
+    end
+
+    attr_reader :key
+    attr_reader :direct_store
+
+    def initialize(key)
+      if(key.respond_to?(:sort))
+        key = key.flatten if key.respond_to?(:flatten)
+        key = key.map(&:to_s).sort
+      end
+      @key = Digest::SHA256.hexdigest(key.to_s)
+      @direct_store = {}
+      @apply_limit = self.class.default_limits
+    end
+
+    def init(name, kind, args={})
+      name = name.to_sym
+      unless(@direct_store[name])
+        full_name = [key, name.to_s].join('_')
+        @direct_store[name] = get_storage(self.class.type, kind, full_name, args)
+      end
+      true
+    end
+
+    def clear!(*args)
+      internal_lock do
+        args = @direct_store.keys if args.empty?
+        args.each do |key|
+          value = @direct_store[key]
+          if(value.respond_to?(:clear))
+            value.clear
+          elsif(value.respond_to?(:value))
+            value.value = nil
+          end
+        end
+        yield if block_given?
+      end
+      true
+    end
+
+    def get_storage(store_type, data_type, full_name, args={})
+      case store_type.to_sym
+      when :redis
+        get_redis_storage(data_type, full_name, args)
+      when :local
+        get_local_storage(data_type, full_name, args)
+      else
+        raise TypeError.new("Unsupported caching storage type encountered: #{store_type}")
+      end
+    end
+
+    def get_redis_storage(data_type, full_name, args={})
+      case data_type.to_sym
+      when :array
+        Redis::List.new(full_name, {:marshal => true}.merge(args))
+      when :hash
+        Redis::HashKey.new(full_name)
+      when :value
+        Redis::Value.new(full_name, {:marshal => true}.merge(args))
+      when :lock
+        Redis::Lock.new(full_name, {:expiration => 3, :timeout => 0.1}.merge(args))
+      when :stamped
+        Stamped.new(full_name.sub("#{key}_", '').to_sym, get_redis_storage(:value, full_name), self)
+      else
+        raise TypeError.new("Unsupported caching data type encountered: #{data_type}")
+      end
+    end
+
+    def get_local_storage(data_type, full_name, args={})
+      case data_type.to_sym
+      when :array
+        []
+      when :hash
+        {}
+      when :value
+        LocalValue.new
+      when :lock
+        LocalLock.new
+      when :stamped
+        Stamped.new(full_name.sub("#{key}_", '').to_sym, get_local_storage(:value, full_name), self)
+      else
+        raise TypeError.new("Unsupported caching data type encountered: #{data_type}")
+      end
+    end
+
+    def internal_lock
+      get_storage(self.class.type, :lock, :internal_access, :timeout => 20).lock do
+        yield
+      end
+    end
+
+    def [](name)
+      internal_lock do
+        @direct_store[name.to_sym]
+      end
+    end
+
+    def []=(key, val)
+      raise 'Setting backend data is not allowed'
+    end
+
+    def time_check_allow?(key, stamp)
+      Time.now.to_i - stamp.to_i > apply_limit(key)
+    end
+
+    def apply_limit(kind, seconds=nil)
+      @apply_limit ||= {}
+      if(seconds)
+        @apply_limit[kind.to_sym] = seconds.to_i
+      end
+      @apply_limit[kind.to_sym].to_i
+    end
+
+
+    class LocalValue
+      attr_accessor :value
+      def initialize(*args)
+        @value = nil
+      end
+    end
+
+    class LocalLock
+      def initialize(*args)
+      end
+
+      def lock
+        yield
+      end
+
+      def clear
+      end
+    end
+
+    class Stamped
+
+      def initialize(name, base, cache)
+        @name = name.to_sym
+        @base = base
+        @cache = cache
+      end
+
+      def value
+        @base.value[:value] if set?
+      end
+
+      def value=(v)
+        @base.value = {:stamp => Time.now.to_i, :value => v}
+      end
+
+      def set?
+        @base.value.is_a?(Hash)
+      end
+
+      def stamp
+        @base.value[:stamp] if set?
+      end
+
+      def update_allowed?
+        !set? || @cache.time_check_allow?(@name, @base.value[:stamp])
+      end
+    end
+
+  end
+end

data/lib/knife-cloudformation/utils.rb

@@ -1,5 +1,23 @@
 module KnifeCloudformation
   module Utils
+
+    module Debug
+      module Output
+        def debug(msg)
+          puts "<KnifeCloudformation>: #{msg}" if ENV['DEBUG']
+        end
+      end
+
+      class << self
+        def included(klass)
+          klass.class_eval do
+            include Output
+            extend Output
+          end
+        end
+      end
+    end
+
     module JSON
 
       def try_json_compat

data/lib/knife-cloudformation/version.rb

@@ -2,5 +2,5 @@ module KnifeCloudformation
   class Version < Gem::Version
   end
 
-  VERSION = Version.new('0.1.12')
+  VERSION = Version.new('0.1.14')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: knife-cloudformation
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.14
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-18 00:00:00.000000000 Z
+date: 2013-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: net-sftp
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sparkle_formation
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+- !ruby/object:Gem::Dependency
+  name: redis-objects
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: attribute_struct
   requirement: !ruby/object:Gem::Requirement
@@ -66,7 +98,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: 0.1.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +106,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: 0.1.8
 description: Knife tooling for Cloud Formation
 email: chrisroberts.code@gmail.com
 executables: []
@@ -90,13 +122,13 @@ files:
 - lib/chef/knife/cloudformation_update.rb
 - lib/chef/knife/cloudformation_inspect.rb
 - lib/knife-cloudformation.rb
-- lib/knife-cloudformation/sparkle_attribute.rb
 - lib/knife-cloudformation/version.rb
 - lib/knife-cloudformation/aws_commons.rb
-- lib/knife-cloudformation/sparkle_formation.rb
+- lib/knife-cloudformation/cache.rb
 - lib/knife-cloudformation/utils.rb
 - lib/knife-cloudformation/aws_commons/stack_parameter_validator.rb
 - lib/knife-cloudformation/aws_commons/stack.rb
+- Gemfile
 - README.md
 - knife-cloudformation.gemspec
 - CHANGELOG.md

data/lib/knife-cloudformation/sparkle_attribute.rb

@@ -1,67 +0,0 @@
-require 'attribute_struct'
-
-module SparkleAttribute
-
-  # TODO: look at the docs for Fn stuff. We can probably just map
-  # simple ones with a bit of string manipulations
-  
-  def _cf_join(*args)
-    options = args.detect{|i| i.is_a?(Hash) && i[:options]} || {:options => {}}
-    args.delete(options)
-    unless(args.size == 1)
-      args = [args]
-    end
-    {'Fn::Join' => [options[:options][:delimiter] || '', *args]}
-  end
-  
-  def _cf_ref(thing)
-    thing = _process_key(thing, :force) if thing.is_a?(Symbol)
-    {'Ref' => thing}
-  end
-
-  def _cf_map(thing, key, *suffix)
-    suffix = suffix.map do |item|
-      if(item.is_a?(Symbol))
-        _process_key(item, :force)
-      else
-        item
-      end
-    end
-    thing = _process_key(thing, :force) if thing.is_a?(Symbol)
-    key = _process_key(key, :force) if key.is_a?(Symbol)
-    {'Fn::FindInMap' => [_process_key(thing), {'Ref' => _process_key(key)}, *suffix]}
-  end
-
-  def _cf_attr(*args)
-    args = args.map do |thing|
-      if(thing.is_a?(Symbol))
-        _process_key(thing, :force)
-      else
-        thing
-      end
-
-    end
-    {'Fn::GetAtt' => args}
-  end
-
-  def _cf_base64(arg)
-    {'Fn::Base64' => arg}
-  end
-
-  def rhel?
-    !!@platform[:rhel]
-  end
-
-  def debian?
-    !!@platform[:debian]
-  end
-
-  def _platform=(plat)
-    @platform || __hashish
-    @platform.clear
-    @platform[plat.to_sym] = true
-  end
-  
-end
-
-AttributeStruct.send(:include, SparkleAttribute)

data/lib/knife-cloudformation/sparkle_formation.rb

@@ -1,132 +0,0 @@
-require 'chef/mash'
-require 'attribute_struct'
-require 'knife-cloudformation/sparkle_attribute'
-require 'knife-cloudformation/utils'
-
-AttributeStruct.camel_keys = true
-
-module KnifeCloudformation
-  class SparkleFormation
-
-    include KnifeCloudformation::Utils::AnimalStrings
-
-    class << self
-
-      attr_reader :dynamics
-      attr_reader :components_path
-      attr_reader :dynamics_path
-
-      def custom_paths
-        @_paths ||= {}
-        @_paths
-      end
-
-      def components_path=(path)
-        custom_paths[:sparkle_path] = path
-      end
-
-      def dynamics_path=(path)
-        custom_paths[:dynamics_directory] = path
-      end
-
-      def compile(path)
-        formation = self.instance_eval(IO.read(path), path, 1)
-        formation.compile._dump
-      end
-
-      def build(&block)
-        struct = AttributeStruct.new
-        struct.instance_exec(&block)
-        struct
-      end
-
-      def load_component(path)
-        self.instance_eval(IO.read(path), path, 1)
-      end
-
-      def load_dynamics!(directory)
-        @loaded_dynamics ||= []
-        Dir.glob(File.join(directory, '*.rb')).each do |dyn|
-          dyn = File.expand_path(dyn)
-          next if @loaded_dynamics.include?(dyn)
-          self.instance_eval(IO.read(dyn), dyn, 1)
-          @loaded_dynamics << dyn
-        end
-        @loaded_dynamics.uniq!
-        true
-      end
-
-      def dynamic(name, &block)
-        @dynamics ||= Mash.new
-        @dynamics[name] = block
-      end
-
-      def insert(dynamic_name, struct, *args)
-        if(@dynamics && @dynamics[dynamic_name])
-          struct.instance_exec(*args, &@dynamics[dynamic_name])
-          struct
-        else
-          raise "Failed to locate requested dynamic block for insertion: #{dynamic_name} (valid: #{@dynamics.keys.sort.join(', ')})"
-        end
-      end
-
-      def from_hash(hash)
-        struct = AttributeStruct.new
-        struct._camel_keys_set(:auto_discovery)
-        struct._load(hash)
-        struct._camel_keys_set(nil)
-        struct
-      end
-    end
-
-    attr_reader :name
-    attr_reader :sparkle_path
-    attr_reader :components
-    attr_reader :load_order
-
-    def initialize(name, options={})
-      @name = name
-      @sparkle_path = options[:sparkle_path] ||
-        self.class.custom_paths[:sparkle_path] ||
-        File.join(Dir.pwd, 'cloudformation/components')
-      @dynamics_directory = options[:dynamics_directory] ||
-        self.class.custom_paths[:dynamics_directory] ||
-        File.join(File.dirname(@sparkle_path), 'dynamics')
-      self.class.load_dynamics!(@dynamics_directory)
-      @components = Mash.new
-      @load_order = []
-    end
-
-    def load(*args)
-      args.each do |thing|
-        if(thing.is_a?(Symbol))
-          path = File.join(sparkle_path, "#{thing}.rb")
-        else
-          path = thing
-        end
-        key = File.basename(path).sub('.rb', '')
-        components[key] = self.class.load_component(path)
-        @load_order << key
-      end
-      self
-    end
-
-    def overrides(&block)
-      @overrides = self.class.build(&block)
-      self
-    end
-
-    # Returns compiled Mash instance
-    def compile
-      compiled = AttributeStruct.new
-      @load_order.each do |key|
-        compiled._merge!(components[key])
-      end
-      if(@overrides)
-        compiled._merge!(@overrides)
-      end
-      compiled
-    end
-
-  end
-end