knife-cloudformation 0.1.18 → 0.1.20

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.1.20
+* Update some caching behavior
+* Add more logging especially around remote calls
+* Add support for request throttling
+* Disable local caching when stack is in `in_progress` state
+
 ## v0.1.18
 * Replace constant with inline value to prevent warnings
 * Explicitly load file to ensure proper load ordering

data/lib/knife-cloudformation/aws_commons.rb

@@ -9,6 +9,20 @@ end
 module KnifeCloudformation
   class AwsCommons
 
+    class << self
+      def logger=(l)
+        @logger = l
+      end
+
+      def logger
+        unless(@logger)
+          require 'logger'
+          @logger = Logger.new($stdout)
+        end
+        @logger
+      end
+    end
+
     include KnifeCloudformation::Utils::AnimalStrings
     include KnifeCloudformation::Utils::Debug
 
@@ -21,11 +35,21 @@ module KnifeCloudformation
     def initialize(args={})
       @ui = args[:ui]
       @credentials = @creds = args[:fog]
+      @disconnect_long_jobs = args[:disconnect_long_jobs]
+      @throttled = []
       @connections = {}
       @memo = Cache.new(credentials)
       @local = {:stacks => {}}
     end
 
+    def logger
+      @logger || self.class.logger
+    end
+
+    def logger=(l)
+      @logger = l
+    end
+
     def cache
       @memo
     end
@@ -63,7 +87,13 @@ module KnifeCloudformation
           end
         end
       end
-      @connections[type]
+      if(block_given?)
+        throttleable do
+          yield @connections[type]
+        end
+      else
+        @connections[type]
+      end
     end
     alias_method :aws, :build_connection
 
@@ -89,9 +119,17 @@ module KnifeCloudformation
           cache.apply_limit(:stacks, args[:refresh_every].to_i)
         end
         if(@memo[:stacks].update_allowed? || args[:force_refresh])
-          @memo[:stacks_lock].lock do
-            @_stacks_cached = true
-            @memo[:stacks].value = aws(:cloud_formation).describe_stacks.body['Stacks']
+          long_running_job(:stacks) do
+            logger.debug 'Populating full cloudformation list from remote end point'
+            stack_result = throttleable do
+              aws(:cloud_formation).describe_stacks.body['Stacks']
+            end
+            if(stack_result)
+              @memo[:stacks_lock].lock do
+                @memo[:stacks].value = stack_result
+              end
+            end
+            logger.debug 'Full cloudformation list from remote end point complete'
           end
         end
       end
@@ -100,6 +138,51 @@ module KnifeCloudformation
       end
     end
 
+    def throttleable
+      if(@throttled.size > 0)
+        if(Time.now.to_i - @throttled.last < Time.now.to_i - @throttled.size * 15)
+          logger.error "Currently being throttled. Not running request!"
+          return nil
+        end
+      end
+      begin
+        result = yield
+        @throttled.clear
+        result
+      rescue Fog::Service::Error => e
+        if(e.message == 'Throttling => Rate exceeded')
+          logger.error "Remote end point is is currently throttling. Rate has been exceeded."
+          @throttled << Time.now.to_i
+        end
+        nil
+      end
+    end
+
+    def long_running_job(name)
+      if(@disconnect_long_jobs)
+        logger.debug "Disconnected long running jobs enabled. Starting: #{name}"
+        lock_key = "long_jobs_lock_#{name}".to_sym
+        @memo.init(lock_key, :lock)
+        Thread.new do
+          begin
+            @memo[lock_key].lock do
+              begin
+                logger.info "Long running job started disconnected (#{name})"
+                yield
+              rescue => e
+                logger.error "Long running job failure (#{name}): #{e.class} - #{e}\n#{e.backtrace.join("\n")}"
+              end
+            end
+          rescue Redis::Lock::Timeout
+            logger.warn "Long running process failed to aquire lock. Request not run (#{name})"
+          end
+        end
+      else
+        logger.debug "Disconnected long running jobs disabled. Starting #{name} inline"
+        yield
+      end
+    end
+
     def name_from_stack_id(s_id)
       found = stacks.detect do |s|
         s['StackId'] == s_id
@@ -115,15 +198,21 @@ module KnifeCloudformation
     end
 
     def stack(*names)
+      direct_load = names.delete(:ignore_seeds)
       result = names.map do |name|
         [name, name.start_with?('arn:') ? name : id_from_stack_name(name)]
       end.map do |name, s_id|
         unless(@local[:stacks][s_id])
-          if(@_stacks_cached)
+          unless(direct_load)
             seed = stacks.detect do |stk|
               stk['StackId'] == s_id
             end
           end
+          if(seed)
+            logger.debug "Requested stack (#{name}) loaded via cached seed"
+          else
+            logger.debug "Requested stack (#{name}) loaded directly with no seed"
+          end
           @local[:stacks][s_id] = Stack.new(name, self, seed)
         end
         @local[:stacks][s_id]

data/lib/knife-cloudformation/aws_commons/stack.rb

@@ -399,14 +399,15 @@ module KnifeCloudformation
           as_resources = resources.find_all do |r|
             r['ResourceType'] == 'AWS::AutoScaling::AutoScalingGroup'
           end
-          @local[:nodes] = as_resources.map do |as_resource|
+          value = as_resources.map do |as_resource|
             as_group = expand_resource(as_resource)
             as_group.instances.map do |inst|
               common.aws(:ec2).servers.get(inst.id)
             end
           end.flatten
+          @local[:nodes] = value unless in_progress?
         end
-        @local[:nodes]
+        value || @local[:nodes]
       end
 
       def nodes_data(*args)
@@ -420,9 +421,9 @@ module KnifeCloudformation
           end
         end
         unless(data.empty?)
-          @memo[cache_key].value = data
+          @memo[cache_key].value = data unless in_progress?
         end
-        @memo[cache_key].value || data
+        data || @memo[cache_key].value
       end
 
     end

data/lib/knife-cloudformation/cloudformation_base.rb

@@ -22,7 +22,7 @@ module KnifeCloudformation
       end
 
       def stack(name)
-        self.class.con(ui).stack(name)
+        self.class.con(ui).stack(name, :ignore_seeds)
       end
 
       def allowed_attributes

data/lib/knife-cloudformation/export.rb

@@ -7,7 +7,8 @@ module KnifeCloudformation
     DEFAULT_OPTIONS = {
       :chef_popsicle => true,
       :ignored_parameters => ['Environment', 'StackCreator'],
-      :chef_environment_parameter => 'Environment'
+      :chef_environment_parameter => 'Environment',
+      :aws_commons => nil
     }
 
     attr_reader :stack, :stack_name, :stack_id, :options, :aws_commons

data/lib/knife-cloudformation/version.rb

@@ -2,5 +2,5 @@ module KnifeCloudformation
   class Version < Gem::Version
   end
 
-  VERSION = Version.new('0.1.18')
+  VERSION = Version.new('0.1.20')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: knife-cloudformation
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-13 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -152,6 +152,7 @@ files:
 - knife-cloudformation.gemspec
 - CHANGELOG.md
 - Gemfile.lock
+- knife-cloudformation-0.1.18.gem
 homepage: http://github.com/heavywater/knife-cloudformation
 licenses: []
 post_install_message: