dynamo-autoscale 0.2.9 → 0.3

data/CHANGELOG

@@ -1,3 +1,11 @@
+v0.3
+
+- Added backdating on startup. When dynamo-autoscale starts up, it now fetches 6
+	hours of data for each table then starts polling.
+- Added a check before actioning that makes sure CloudWatch and DynamoDB agree
+	on what the current provisioned throughput is. Fixes a bug where downscales
+	were happening too frequently.
+
 v0.2.9
 
 - Fixed a problem where scale events weren't being showed in the scale report

data/bin/dynamo-autoscale

@@ -33,5 +33,8 @@ unless DynamoAutoscale.config[:dry_run]
   DynamoAutoscale.actioner_class = DynamoAutoscale::DynamoActioner
 end
 
-DynamoAutoscale.logger.info "Finished setup. Starting polling loop."
+DynamoAutoscale.logger.info "Finished setup. Backdating..."
+DynamoAutoscale.poller.backdate
+
+DynamoAutoscale.logger.info "Starting polling loop..."
 DynamoAutoscale.poller.run

data/lib/dynamo-autoscale/actioner.rb

@@ -245,13 +245,26 @@ module DynamoAutoscale
       return result
     end
 
+    def pending_reads?
+      !!@pending[:reads]
+    end
+
+    def pending_writes?
+      !!@pending[:writes]
+    end
+
+    def clear_pending!
+      @pending[:writes] = nil
+      @pending[:reads] = nil
+    end
+
     def should_flush?
       if @opts[:group_downscales].nil?
         logger.info "[flush] Downscales are not being grouped. Should flush."
         return true
       end
 
-      if @pending[:reads] and @pending[:writes]
+      if pending_reads? and pending_writes?
         logger.info "[flush] Both a read and a write are pending. Should flush."
         return true
       end

data/lib/dynamo-autoscale/cw_poller.rb

@@ -3,10 +3,21 @@ module DynamoAutoscale
     include DynamoAutoscale::Logger
     INTERVAL = 5.minutes
 
-    def poll tables, &block
-      if tables.nil?
-        tables = AWS::DynamoDB.new.tables.to_a.map(&:name)
+    def backdate
+      now = Time.now.utc
+
+      @tables.each do |table_name|
+        table = DynamoAutoscale.tables[table_name]
+        dispatch(table, Metrics.all_metrics(table_name, {
+          period:     5.minutes,
+          start_time: now - 6.hours,
+          end_time:   now,
+        }))
       end
+    end
+
+    def poll tables, &block
+      tables = AWS::DynamoDB.new.tables.to_a.map(&:name) if tables.nil?
 
       loop do
         # Sleep until the next interval occurrs. This calculation ensures that
@@ -24,7 +35,7 @@ module DynamoAutoscale
       logger.debug "[cw_poller] Beginning CloudWatch poll..."
       now = Time.now
 
-      tables.each do |table|
+      tables.each do |table_name|
         # This code will dispatch a message to the listening table that looks
         # like this:
         #
@@ -39,7 +50,7 @@ module DynamoAutoscale
         #
         # There may also be :provisioned_reads and :provisioned_writes
         # depending on how the CloudWatch API feels.
-        block.call(table, Metrics.all_metrics(table, {
+        block.call(table_name, Metrics.all_metrics(table_name, {
           period:     5.minutes,
           start_time: now - 20.minutes,
           end_time:   now,

data/lib/dynamo-autoscale/dispatcher.rb

@@ -33,13 +33,18 @@ module DynamoAutoscale
       block.call(table, time, datum) if block
 
       if @last_check[table.name].nil? or @last_check[table.name] < time
-        if DynamoAutoscale.actioners[table].can_run?
-          logger.debug "[dispatcher] Checking rules..."
-          DynamoAutoscale.rules.test(table)
-          @last_check[table.name] = time
+        if time > 20.minutes.ago # Too young to vote!
+          if DynamoAutoscale.actioners[table].can_run?
+            logger.debug "[dispatcher] Checking rules..."
+            DynamoAutoscale.rules.test(table)
+            @last_check[table.name] = time
+          else
+            logger.debug "[dispatcher] Skipped rule check, table is not ready " +
+              "to have its throughputs modified."
+          end
         else
-          logger.debug "[dispatcher] Skipped rule check, table is not ready " +
-            "to have its throughputs modified."
+          logger.debug "[dispatcher] Skipped rule check, data point was from " +
+            "more than 20 minutes ago. Too young to vote!"
         end
       else
         logger.debug "[dispatcher] Skipped rule check, already checked for " +

data/lib/dynamo-autoscale/dynamo_actioner.rb

@@ -14,17 +14,57 @@ module DynamoAutoscale
         :write_capacity_units
       end
 
-      dynamo_scale(aws_throughput_key => value)
+      if throughput_synced?
+        dynamo_scale(aws_throughput_key => value)
+      else
+        # If the throughputs were not synced, the likelihood is we made the
+        # decision to scale based on false data. Clear it.
+        clear_pending!
+        false
+      end
     end
 
     def scale_both reads, writes
-      dynamo_scale(read_capacity_units: reads, write_capacity_units: writes)
+      if throughput_synced?
+        dynamo_scale(read_capacity_units: reads, write_capacity_units: writes)
+      else
+        # If the throughputs were not synced, the likelihood is we made the
+        # decision to scale based on false data. Clear it.
+        clear_pending!
+        false
+      end
     end
 
     def can_run?
       dynamo.status == :active
     end
 
+    def throughput_synced?
+      time, datum = table.data.last
+
+      # If we've not gathered any data, we cannot know if our values are synced
+      # with Dynamo so we have to assume they are not until we get some data in.
+      return false if time.nil? or datum.nil?
+
+      if dynamo.read_capacity_units != datum[:provisioned_reads]
+        logger.error "[actioner] DynamoDB disagrees with CloudWatch on what " +
+          "the provisioned reads are. To be on the safe side, operations are " +
+          "not being applied."
+
+        return false
+      end
+
+      if dynamo.write_capacity_units != datum[:provisioned_writes]
+        logger.error "[actioner] DynamoDB disagrees with CloudWatch on what " +
+          "the provisioned writes are. To be on the safe side, operations are " +
+          "not being applied."
+
+        return false
+      end
+
+      return true
+    end
+
     private
 
     def dynamo_scale opts

data/lib/dynamo-autoscale/poller.rb

@@ -19,24 +19,27 @@ module DynamoAutoscale
     def run &block
       poll(tables) do |table_name, data|
         logger.debug "[poller] Got data: #{data}"
-        table = DynamoAutoscale.tables[table_name]
 
-        times = data.inject([]) do |memo, (_, timeseries)|
-          memo += timeseries.keys
-        end.sort!.uniq!
+        dispatch(DynamoAutoscale.tables[table_name], data, &block)
+      end
+    end
+
+    def dispatch table, data, &block
+      times = data.inject([]) do |memo, (_, timeseries)|
+        memo += timeseries.keys
+      end.sort!.uniq!
 
-        times.each do |time|
-          datum = {
-            provisioned_writes: data[:provisioned_writes][time],
-            provisioned_reads:  data[:provisioned_reads][time],
-            consumed_writes:    data[:consumed_writes][time],
-            consumed_reads:     data[:consumed_reads][time],
-          }
+      times.each do |time|
+        datum = {
+          provisioned_writes: data[:provisioned_writes][time],
+          provisioned_reads:  data[:provisioned_reads][time],
+          consumed_writes:    data[:consumed_writes][time],
+          consumed_reads:     data[:consumed_reads][time],
+        }
 
-          filters.each { |filter| filter.call(table, time, datum) }
+        filters.each { |filter| filter.call(table, time, datum) }
 
-          DynamoAutoscale.dispatcher.dispatch(table, time, datum, &block)
-        end
+        DynamoAutoscale.dispatcher.dispatch(table, time, datum, &block)
       end
     end
   end

data/lib/dynamo-autoscale/version.rb

@@ -1,3 +1,3 @@
 module DynamoAutoscale
-  VERSION = '0.2.9'
+  VERSION = '0.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamo-autoscale
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: '0.3'
   prerelease: 
 platform: ruby
 authors: