logstash-input-cloudwatch_logs 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49516d63ed0fba659dbe33edc4288c61c084659b
-  data.tar.gz: 6004ebd1359d675c0bc166935d04038e530b3902
+  metadata.gz: 46906c7db7628771d3487566ba11d1987e05aa29
+  data.tar.gz: 5410ad160767b02f7390069b6461d937b51b789c
 SHA512:
-  metadata.gz: 4ccdac70044254d0a8e2aadd333473a5eb5df37928507d5416975c915ab9905501f28601f2449519ea5e539f72eba7c3d3195938d165cb58acf349e33e9f8370
-  data.tar.gz: c49abd01f34cdedd7448551f1d4d54fc597dade50e0e69de54afed29bef468e5424a72f8c23c16cbbf098b85a0ce08862027be99ef1a634290add64707d3e02d
+  metadata.gz: fe06db80935feed58856787ed6ef888ce566b5a3ff340cf0c7fe2e3db602db072ab6a83349b41a6d011bd06a62db931e809438876397bc0571d1b553e895761e
+  data.tar.gz: a47d4a612dae319daf23a1b30a5ece6c00e2ea2dec6e041806629687c8d15b1bf9da3be4ca86ce02946de4fc8ecdfafede89a9fe113607a4c6e709629c9d5005

data/CHANGELOG.md

@@ -5,6 +5,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [v1.0.1] (2017-08-22)
+
+### Fixed
+* Handle rate limiting when making requests to the CWL apis ([#32](https://github.com/lukewaite/logstash-input-cloudwatch-logs/pull/32))
+
 ## [v1.0.0] (2017-07-15)
 
 ### Added
@@ -18,5 +23,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * Pull only log_events since last ingestion (Fixes [#10](https://github.com/lukewaite/logstash-input-cloudwatch-logs/issues/10))
 * Incrementally write to since_db on each page of data from the CWL API (Fixes [#4](https://github.com/lukewaite/logstash-input-cloudwatch-logs/issues/4))
 
-[Unreleased]: https://github.com/lukewaite/logstash-input-cloudwatch-logs/compare/v1.0.0...HEAD
+[Unreleased]: https://github.com/lukewaite/logstash-input-cloudwatch-logs/compare/v1.0.1...HEAD
+[v1.0.1]: https://github.com/lukewaite/logstash-input-cloudwatch-logs/compare/v0.1.0.0...v1.0.1
 [v1.0.0]: https://github.com/lukewaite/logstash-input-cloudwatch-logs/compare/v0.10.3...v1.0.0

data/lib/logstash/inputs/cloudwatch_logs.rb

@@ -110,16 +110,21 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
   public
   def run(queue)
     @queue = queue
+    @priority = []
     _sincedb_open
     determine_start_position(find_log_groups)
 
     while !stop?
-      groups = find_log_groups
-
-      groups.each do |group|
-        @logger.debug("calling process_group on #{group}")
-        process_group(group)
-      end # groups.each
+      begin
+        groups = find_log_groups
+
+        groups.each do |group|
+          @logger.debug("calling process_group on #{group}")
+          process_group(group)
+        end # groups.each
+      rescue Aws::CloudWatchLogs::Errors::ThrottlingException
+        @logger.debug("reached rate limit")
+      end
 
       Stud.stoppable_sleep(@interval) { stop? }
     end
@@ -144,9 +149,15 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
       @logger.debug("log_group_prefix not enabled")
       groups = @log_group
     end
-    groups
+    # Move the most recent groups to the end
+    groups.sort{|a,b| priority_of(a) <=> priority_of(b) }
   end # def find_log_groups
 
+  private
+  def priority_of(group)
+    @priority.index(group) || -1
+  end
+
   public
   def determine_start_position(groups)
     groups.each do |group|
@@ -189,6 +200,8 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
       next_token = resp.next_token
       break if next_token.nil?
     end
+    @priority.delete(group)
+    @priority << group
   end #def process_group
 
   # def process_log

data/logstash-input-cloudwatch_logs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-cloudwatch_logs'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Stream events from CloudWatch Logs.'
   s.description     = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudwatch_logs
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Luke Waite
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-15 00:00:00.000000000 Z
+date: 2017-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement