fluent-plugin-cloudwatch-logs 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5082b725cf548ffea0fb7e779c6c24a4d7bb7909
-  data.tar.gz: 6a8dbc8986550ec45e9fb3518316f19d7267ac8e
+  metadata.gz: 177bb1a6b7af9c6565f1cd5104f422c1fb48add7
+  data.tar.gz: 576f39a7a7cec61aa655aefbc60549f1787f9f77
 SHA512:
-  metadata.gz: f8321eeaa7a481dae684a5908446782a5ba570b1bcc9bbeec86a63d4a365340923a4478536a471a4480aa32b6017a537c5023ee87dd5d0144d62cab252f0c6fd
-  data.tar.gz: c57ab7ca6be0756ad4377b3b7cf12cc69b6f4e141b82c5c5cc664fe493fe22f705758c5e3c4be2d00fcec55d98c892c8c325c3f17b453acd899b7ccf266b01cb
+  metadata.gz: 998c9861c7db2ed024ef27c2011a70a52a8648797f2b5f5891381f6c5620b2d4074279b0e682b2123f0f6f9a90d197f388943fec9416271c6f045261d39b326e
+  data.tar.gz: 900de6d31aeb5e62f6695c9ae016e16470ce0c550cb2e9b0851d03a073095862808dab9093a22586709ff05310445c47c7134c89b8a4485c0032ef69e8998dbe

data/README.md

@@ -79,6 +79,9 @@ Fetch sample log from CloudWatch Logs:
   #log_stream_name_key stream_name_key
   #remove_log_group_name_key true
   #remove_log_stream_name_key true
+  #put_log_events_retry_wait 1s
+  #put_log_events_retry_limit 17
+  #put_log_events_disable_retry_limit false
 </match>
 ```
 
@@ -96,6 +99,9 @@ Fetch sample log from CloudWatch Logs:
 * `log_stream_name_key`: use specified field of records as log stream name
 * `remove_log_group_name_key`: remove field specified by `log_group_name_key`
 * `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
+* `put_log_events_retry_wait`: time before retrying PutLogEvents (retry interval increases exponentially like `put_log_events_retry_wait * (2 ^ retry_count)`)
+* `put_log_events_retry_limit`: maximum count of retry (if exceeding this, the events will be discarded)
+* `put_log_events_disable_retry_limit`: if true, `put_log_events_retry_limit` will be ignored
 
 ### in_cloudwatch_logs
 

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "mocha"
 end

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.2.3"
+        VERSION = "0.2.4"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -23,6 +23,9 @@ module Fluent
     config_param :remove_log_group_name_key, :bool, :default => false
     config_param :remove_log_stream_name_key, :bool, :default => false
     config_param :http_proxy, :string, default: nil
+    config_param :put_log_events_retry_wait, :time, default: 1.0
+    config_param :put_log_events_retry_limit, :integer, default: 17
+    config_param :put_log_events_disable_retry_limit, :bool, default: false
 
     MAX_EVENTS_SIZE = 1_048_576
     EVENT_HEADER_SIZE = 26
@@ -60,7 +63,7 @@ module Fluent
       options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       options[:region] = @region if @region
       options[:http_proxy] = @http_proxy if @http_proxy
-      @logs = Aws::CloudWatchLogs::Client.new(options)
+      @logs ||= Aws::CloudWatchLogs::Client.new(options)
       @sequence_tokens = {}
     end
 
@@ -191,14 +194,42 @@ module Fluent
       token = next_sequence_token(group_name, stream_name)
       args[:sequence_token] = token if token
 
-      log.debug "Calling PutLogEvents API", {
-        "group" => group_name,
-        "stream" => stream_name,
-        "events_count" => events.size,
-        "events_bytesize" => events_bytesize,
-      }
+      response = nil
+      retry_count = 0
+      until response
+        log.debug "Calling PutLogEvents API", {
+          "group" => group_name,
+          "stream" => stream_name,
+          "events_count" => events.size,
+          "events_bytesize" => events_bytesize,
+        }
+        begin
+          response = @logs.put_log_events(args)
+        rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
+          if !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
+            log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
+              "error_class" => err.class.to_s,
+              "error" => err.message,
+            }
+            return
+          else
+            sleep_sec = @put_log_events_retry_wait * (2 ** retry_count)
+            sleep_sec += sleep_sec * (0.25 * (rand - 0.5))
+            log.warn "failed to PutLogEvents", {
+              "next_retry" => Time.now + sleep_sec,
+              "error_class" => err.class.to_s,
+              "error" => err.message,
+            }
+            sleep(sleep_sec)
+            retry_count += 1
+          end
+        end
+      end
+
+      if 0 < retry_count
+        log.warn "retry succeeded"
+      end
 
-      response = @logs.put_log_events(args)
       store_next_sequence_token(group_name, stream_name, response.next_sequence_token)
     end
 

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -291,6 +291,49 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
   end
 
+  def test_retrying_on_throttling_exception
+    resp = mock()
+    resp.expects(:next_sequence_token)
+    client = Aws::CloudWatchLogs::Client.new
+    client.stubs(:put_log_events).
+      raises(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")).then.returns(resp)
+
+    time = Time.now
+    d = create_driver
+    d.instance.instance_variable_set(:@logs, client)
+    d.emit({'message' => 'message1'}, time.to_i)
+    d.run
+
+    assert_match(/Calling PutLogEvents/, d.instance.log.logs[0])
+    assert_match(/failed to PutLogEvents/, d.instance.log.logs[1])
+    assert_match(/Calling PutLogEvents/, d.instance.log.logs[2])
+    assert_match(/retry succeeded/, d.instance.log.logs[3])
+  end
+
+  def test_retrying_on_throttling_exception_and_throw_away
+    client = Aws::CloudWatchLogs::Client.new
+    client.stubs(:put_log_events).
+      raises(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
+
+    time = Time.now
+    d = create_driver(<<-EOC)
+#{default_config}
+log_group_name #{log_group_name}
+log_stream_name #{log_stream_name}
+put_log_events_retry_limit 1
+    EOC
+    d.instance.instance_variable_set(:@logs, client)
+    d.emit({'message' => 'message1'}, time.to_i)
+    d.run
+
+    assert_match(/Calling PutLogEvents/, d.instance.log.logs[0])
+    assert_match(/failed to PutLogEvents/, d.instance.log.logs[1])
+    assert_match(/Calling PutLogEvents/, d.instance.log.logs[2])
+    assert_match(/failed to PutLogEvents/, d.instance.log.logs[3])
+    assert_match(/Calling PutLogEvents/, d.instance.log.logs[4])
+    assert_match(/failed to PutLogEvents and throwing away/, d.instance.log.logs[5])
+  end
+
   private
   def default_config
     <<-EOC

data/test/test_helper.rb

@@ -1,4 +1,5 @@
 require 'test/unit'
+require 'mocha/test_unit'
 require 'fluent/test'
 
 require 'aws-sdk-core'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-18 00:00:00.000000000 Z
+date: 2016-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - ryota.arai@gmail.com