fluent-plugin-cloudwatch-logs 0.9.3 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa72486196ed38595f884945613e076fecd8f66c6d77046e118a8fc24301dc06
-  data.tar.gz: 1076cb96661bf6d89d3a958989c04a9f690fae9b5ac6aae33a7d9231e5cbb8d5
+  metadata.gz: 0b2918da30191dd56e19d14043de953c649cb6207226df51a4721b8dda6af464
+  data.tar.gz: a6a6def2e408711e723f6cec2f5fb706981f23479dd8cb9a3a1a674d859c1cbf
 SHA512:
-  metadata.gz: c21965fb1e9c340a92cc2d482ce5fed0590736f14df03da1d6706ae039f74f2e767f897e693fafdba644306b9ab4dde178a11abcbb23fb138b7b64f6b3ba08cc
-  data.tar.gz: 93b8ec018eeffa3229b12f9daf312ec660eafb39d4159c42c92863c0150a3d29c43b93ff55dfda46e4f03917a96eb92ef4f59f358e23eda2e37facb0572ecfce
+  metadata.gz: 3aa61613b097ff349ba56fd9a5b7e0ee4f87ed3fa23bfa816c35decc193d1161a1771b6985c745f751cc0d05e8e0be606326e145ec19a509f1d2ebd2698f7d71
+  data.tar.gz: ffca6e46ee0c87dc22cd60c4cd724bdf22421daaff0338f245436edce33a8f0397f3f08daa4dd0f929cfabb6d73a2718feff15d304b5e79ca6762c0808aebd93

data/README.md

@@ -193,6 +193,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `log_group_name`: name of log group to fetch logs
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
+* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
 * `state_file`: file to store current state (e.g. next\_forward\_token)
 * `tag`: fluentd tag
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.9.3"
+        VERSION = "0.9.4"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -30,6 +30,7 @@ module Fluent::Plugin
     config_param :start_time, :string, default: nil
     config_param :end_time, :string, default: nil
     config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
+    config_param :throttling_retry_seconds, :time, default: nil
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -183,6 +184,15 @@ module Fluent::Plugin
       end
 
       response.events
+    rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
+      if throttling_retry_seconds
+        log.warn "ThrottlingException in get_log_events (#{log_stream_name}). Waiting #{throttling_retry_seconds} seconds to retry."
+        sleep throttling_retry_seconds
+
+        get_events(log_stream_name)
+      else
+        raise err
+      end
     end
 
     def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -28,6 +28,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
         start_time "2019-06-18 00:00:00Z"
         end_time "2020-01-18 00:00:00Z"
         time_range_format "%Y-%m-%d %H:%M:%S%z"
+        throttling_retry_seconds 30
       EOC
 
       assert_equal('test_id', d.instance.aws_key_id)
@@ -43,6 +44,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(1560816000000, d.instance.start_time)
       assert_equal(1579305600000, d.instance.end_time)
       assert_equal("%Y-%m-%d %H:%M:%S%z", d.instance.time_range_format)
+      assert_equal(30, d.instance.throttling_retry_seconds)
     end
 
     test 'invalid time range' do
@@ -608,6 +610,36 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(["test", ((time_ms + 7000) / 1000), { "cloudwatch" => "logs7" }], events[6])
       assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
     end
+
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        state_file /tmp/state
+        fetch_interval 0.1
+        throttling_retry_seconds 0.2
+      CONFIG
+
+      # it will raises the error 2 times
+      counter = 0
+      times = 2
+      stub(@client).get_log_events(anything) {
+        counter += 1
+        counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(events: [], next_forward_token: nil)
+      }
+
+      d = create_driver(config)
+
+      # so, it is expected to valid_next_token once
+      mock(d.instance).valid_next_token(nil, nil).once
+
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
+
+      d.run
+      assert_equal(2, d.logs.select {|l| l =~ /Waiting 0.2 seconds to retry/ }.size)
+    end
   end
 
   private

data/test/test_helper.rb

@@ -42,10 +42,7 @@ module CloudwatchLogsTestHelper
   end
 
   def log_stream_name(log_stream_name_prefix = nil)
-    if !@log_stream_name
-      new_log_stream(log_stream_name_prefix)
-    end
-    @log_stream_name
+    @log_stream_name ||= new_log_stream(log_stream_name_prefix)
   end
 
   def new_log_stream(log_stream_name_prefix = nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-03 00:00:00.000000000 Z
+date: 2020-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd