fluent-plugin-kinesis-firehose 0.1.1 → 0.1.2.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac09d1fd780d1623ec87b31f2af2d780f91a307d
-  data.tar.gz: a420cfd20a9b7a3f531a88f48d58ff6e579a110f
+  metadata.gz: b5a52689533a4ac87d41ce025a97f9caca093dbc
+  data.tar.gz: 9d2bfd0b85ed883067979a1de7b014290f265f56
 SHA512:
-  metadata.gz: f1fc0ab30520e7d4efd0c1799350ecc576215c79d6b120db00537301ea946297b72ec2587b8382bdfdd21dd1f18248d7df150d0df66683f0491df145e9996240
-  data.tar.gz: 7003b63c6188b0b27554a398d82e4e881e03203511c101456af347ae89ddbb49f120cbba404a3acf09fa4b5a4dbd3dc87ca233cbbbac1b221bf35d5473cb1149
+  metadata.gz: 142208dbeccfad34374d724762f01a53ef3743e9204038b9fdb8932831db63adde4e9c8e3e03c44e90d23287f2ae1776ae8a4f472f66c88982da4279d0d28400
+  data.tar.gz: 53da4b07a323773d3ae87eebbe62992ec664430205864ff247524d9d9fcdb3f4cc27ba8a3036c4193e819943df19e84176447d21c8d2cf4e80b61bb1f07dc7dc

data/README.md

@@ -5,6 +5,11 @@ Fluentd output plugin for Amazon Kinesis Firehose.
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis-firehose.svg)](http://badge.fury.io/rb/fluent-plugin-kinesis-firehose)
 [![Build Status](https://travis-ci.org/winebarrel/fluent-plugin-kinesis-firehose.svg)](https://travis-ci.org/winebarrel/fluent-plugin-kinesis-firehose)
 
+## ChangeLog
+
+* `>= 0.1.2`
+  * Add `retries_on_putrecordbatch` option ([PR#2](https://github.com/winebarrel/fluent-plugin-kinesis-firehose/pull/3))
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -35,6 +40,7 @@ Or install it yourself as:
   #aws_sec_key ...
   region us-east-1
   #endpoint ...
+  #retries_on_putrecordbatch 3
 
   #data_key data (default: nil)
 

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -14,16 +14,17 @@ class Fluent::KinesisFirehoseOutput < Fluent::BufferedOutput
     define_method('log') { $log }
   end
 
-  config_param :profile,              :string, :default => nil
-  config_param :credentials_path,     :string, :default => nil
-  config_param :aws_key_id,           :string, :default => nil, :secret => true
-  config_param :aws_sec_key,          :string, :default => nil, :secret => true
-  config_param :region,               :string, :default => nil
-  config_param :endpoint,             :string, :default => nil
-  config_param :http_proxy,           :string, :default => nil
-  config_param :delivery_stream_name, :string
-  config_param :data_key,             :string, :default => nil
-  config_param :append_new_line,      :bool,   :default => true
+  config_param :profile,                   :string,  :default => nil
+  config_param :credentials_path,          :string,  :default => nil
+  config_param :aws_key_id,                :string,  :default => nil, :secret => true
+  config_param :aws_sec_key,               :string,  :default => nil, :secret => true
+  config_param :region,                    :string,  :default => nil
+  config_param :endpoint,                  :string,  :default => nil
+  config_param :http_proxy,                :string,  :default => nil
+  config_param :delivery_stream_name,      :string
+  config_param :data_key,                  :string,  :default => nil
+  config_param :append_new_line,           :bool,    :default => true
+  config_param :retries_on_putrecordbatch, :integer, :default => 3
 
   def initialize
     super
@@ -33,6 +34,7 @@ class Fluent::KinesisFirehoseOutput < Fluent::BufferedOutput
 
   def configure(conf)
     super
+    @sleep_duration = Array.new(@retries_on_putrecordbatch) {|n| ((2 ** n) * scaling_factor) }
   end
 
   def format(tag, time, record)
@@ -54,6 +56,9 @@ class Fluent::KinesisFirehoseOutput < Fluent::BufferedOutput
     }.each_slice(PUT_RECORDS_MAX_COUNT) {|data_list|
       put_records(data_list)
     }
+  rescue => e
+    log.error e.message
+    log.error_backtrace e.backtrace
   end
 
   private
@@ -86,19 +91,54 @@ class Fluent::KinesisFirehoseOutput < Fluent::BufferedOutput
         false
       end
     }.each {|chunk|
-      put_record_batch(chunk)
+      put_record_batch_with_retry(chunk)
     }
   end
 
-  def put_record_batch(data_list)
+  def put_record_batch_with_retry(data_list, retry_count=0)
     records = data_list.map do |data|
       {:data => data}
     end
 
-    client.put_record_batch(
+    response = client.put_record_batch(
       :delivery_stream_name => @delivery_stream_name,
       :records => records
     )
+
+    if response[:failed_put_count] && response[:failed_put_count] > 0
+      failed_records = collect_failed_records(data_list, response)
+
+      if retry_count < @retries_on_putrecordbatch
+        sleep @sleep_duration[retry_count]
+        retry_count += 1
+        log.warn 'Retrying to put records. Retry count: %d' % retry_count
+        put_record_batch_with_retry(failed_records.map{|record| record[:data] }, retry_count)
+      else
+        failed_records.each {|record|
+          log.error 'Could not put record, Error: %s/%s, Record: %s' % [
+            record[:error_code],
+            record[:error_message],
+            record[:data]
+          ]
+        }
+      end
+    end
+  end
+
+  def collect_failed_records(data_list, response)
+    failed_records = []
+
+    response[:request_responses].each_with_index do |record, index|
+      if record[:error_code]
+        failed_records.push(
+          data: data_list[index],
+          error_code: record[:error_code],
+          error_message: record[:error_message]
+        )
+      end
+    end
+
+    failed_records
   end
 
   def client
@@ -127,4 +167,8 @@ class Fluent::KinesisFirehoseOutput < Fluent::BufferedOutput
 
     @client = Aws::Firehose::Client.new(options)
   end
+
+  def scaling_factor
+    0.5 + rand * 0.1
+  end
 end

data/lib/fluent_plugin_kinesis_firehose/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginKinesisFirehose
-  VERSION = '0.1.1'
+  VERSION = '0.1.2.beta'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis-firehose
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2.beta
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-04 00:00:00.000000000 Z
+date: 2015-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -142,12 +142,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin for Amazon Kinesis Firehose.