fluent-plugin-cloudwatch-logs 0.9.4 → 0.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b2918da30191dd56e19d14043de953c649cb6207226df51a4721b8dda6af464
-  data.tar.gz: a6a6def2e408711e723f6cec2f5fb706981f23479dd8cb9a3a1a674d859c1cbf
+  metadata.gz: 13ef3e0d5f4fd4c12c49b16ccbd07db237b7cff26ea272e4ef6337b09b02e975
+  data.tar.gz: 8cefeed014b1842ccce9f15479ad8cc54aa51b976281abb5d3e1c8828239aaa3
 SHA512:
-  metadata.gz: 3aa61613b097ff349ba56fd9a5b7e0ee4f87ed3fa23bfa816c35decc193d1161a1771b6985c745f751cc0d05e8e0be606326e145ec19a509f1d2ebd2698f7d71
-  data.tar.gz: ffca6e46ee0c87dc22cd60c4cd724bdf22421daaff0338f245436edce33a8f0397f3f08daa4dd0f929cfabb6d73a2718feff15d304b5e79ca6762c0808aebd93
+  metadata.gz: 41ca6fa20d0c26955fcbab7f7a5a9fb8c032a5fd09f8676ccaaf5e4dd12694ea74c7c5320590f076405c691d436bc87b820e6bc9b75dc516062b2f3e09d8a377
+  data.tar.gz: 1f5bbd694c5962de73ee1e0ca622d8892437fb82d00b76ed87086feeba49f6d2c3a4904edad81bec88ba2d3ca47701c15cb56395470f19742c16b3168dd870e5

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.9.4"
+        VERSION = "0.9.5"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -170,49 +170,57 @@ module Fluent::Plugin
     end
 
     def get_events(log_stream_name)
-      request = {
-        log_group_name: @log_group_name,
-        log_stream_name: log_stream_name
-      }
-      request.merge!(start_time: @start_time) if @start_time
-      request.merge!(end_time: @end_time) if @end_time
-      log_next_token = next_token(log_stream_name)
-      request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
-      response = @logs.get_log_events(request)
-      if valid_next_token(log_next_token, response.next_forward_token)
-        store_next_token(response.next_forward_token, log_stream_name)
+      throttling_handler('get_log_events') do
+        request = {
+          log_group_name: @log_group_name,
+          log_stream_name: log_stream_name
+        }
+        request.merge!(start_time: @start_time) if @start_time
+        request.merge!(end_time: @end_time) if @end_time
+        log_next_token = next_token(log_stream_name)
+        request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
+        response = @logs.get_log_events(request)
+        if valid_next_token(log_next_token, response.next_forward_token)
+          store_next_token(response.next_forward_token, log_stream_name)
+        end
+
+        response.events
       end
+    end
 
-      response.events
+    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
+      throttling_handler('describe_log_streams') do
+        request = {
+          log_group_name: @log_group_name
+        }
+        request[:next_token] = next_token if next_token
+        request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
+        response = @logs.describe_log_streams(request)
+        if log_streams
+          log_streams.concat(response.log_streams)
+        else
+          log_streams = response.log_streams
+        end
+        if response.next_token
+          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+        end
+        log_streams
+      end
+    end
+
+    def throttling_handler(method_name)
+      yield
     rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
       if throttling_retry_seconds
-        log.warn "ThrottlingException in get_log_events (#{log_stream_name}). Waiting #{throttling_retry_seconds} seconds to retry."
+        log.warn "ThrottlingException #{method_name}. Waiting #{throttling_retry_seconds} seconds to retry."
         sleep throttling_retry_seconds
 
-        get_events(log_stream_name)
+        throttling_handler(method_name) { yield }
       else
         raise err
       end
     end
 
-    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
-      request = {
-        log_group_name: @log_group_name
-      }
-      request[:next_token] = next_token if next_token
-      request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
-      response = @logs.describe_log_streams(request)
-      if log_streams
-        log_streams.concat(response.log_streams)
-      else
-        log_streams = response.log_streams
-      end
-      if response.next_token
-        log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
-      end
-      log_streams
-    end
-
     def valid_next_token(prev_token, next_token)
       next_token && prev_token != next_token.chomp
     end

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -611,7 +611,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
     end
 
-    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException" do
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in get_log_events" do
       config = <<-CONFIG
         tag test
         @type cloudwatch_logs
@@ -634,11 +634,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       # so, it is expected to valid_next_token once
       mock(d.instance).valid_next_token(nil, nil).once
 
+      d.run
+      assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException get_log_events. Waiting 0.2 seconds to retry/ }.size)
+    end
+
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in describe_log_streams" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        use_log_stream_name_prefix true
+        state_file /tmp/state
+        fetch_interval 0.1
+        throttling_retry_seconds 0.2
+      CONFIG
+
+      # it will raises the error 2 times
       log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
-      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
+      counter = 0
+      times = 2
+      stub(@client).describe_log_streams(anything) {
+        counter += 1
+        counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(log_streams: [log_stream], next_token: nil)
+      }
+
+      d = create_driver(config)
 
       d.run
-      assert_equal(2, d.logs.select {|l| l =~ /Waiting 0.2 seconds to retry/ }.size)
+      assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException describe_log_streams. Waiting 0.2 seconds to retry/ }.size)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.5
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd