fluent-plugin-kinesis 1.1.3 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d2e990f8886423c95645780d0e44dc8fa9db2d6
-  data.tar.gz: f37b1fc727a468940b901698364812f55eeacbb3
+  metadata.gz: d8b59ae71478266b7d619c76b71811310f2cf90a
+  data.tar.gz: 4e3049d7c6ec83dc6da5858fee3ddfa8b3e76660
 SHA512:
-  metadata.gz: ef98238e2b6b7a580f9ac67e439efe758bf40d71b23e9efa2d63165c10a3a6015f9ec20cb07294b33396aa393872616a896fc82d6bf4d54d9541ca1c58646506
-  data.tar.gz: 7c833e6fe327b1c320769ce00f3cd30c58adaa1bf7a7ca463f220e02840d06bb0d0a00ffe7b8bcf7d8a2beae26acaa0cad973b6fde21e2d13f6b0845ae446398
+  metadata.gz: 1bf9e15306dbf5f88f5dcdd731d06395821c7a29f1f0d0597075db19fc638f6b2439003ef08326bac4bc9f1bbd0a124f5ced2fef5b83b0c52b81e4f6fdcbb35a
+  data.tar.gz: 1ac91c53effd50cd9ebb947f7be53ec8162f4d736e836c6076149ed4da80ed39ada244ece88fd23923e9daedb1026e35d74f50e564ced387b338d7b0de8223ed

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 1.2.0
+- Feature - Add reduce_max_size_error_message configuration [#127](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/127)
+- Bug fix - Fix empty batch error [#125](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/125)
+
 ## 1.1.3
 
 - Bug fix - Fix issues with fluentd 0.14.12 [#99](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/99)

data/README.md

@@ -122,7 +122,7 @@ The credential provider will be choosen by the steps below:
 
 - Use [**shared_credentials**](#shared_credentials) section if you set it
 - Use [**assume_role_credentials**](#assume_role_credentials) section if you set it
-- Otherwise, default provicder chain:
+- Otherwise, default provider chain:
     - [**aws_key_id**](#aws_key_id) and [**aws_sec_key**](#aws_sec_key)
     - Environment variables (ex. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc.)
     - Default shared credentials (`default` in `~/.aws/credentials`)
@@ -135,7 +135,11 @@ AWS access key id.
 AWS secret access key.
 
 ### shared_credentials
-Use this config section to specify shared credential file path and profile name. If you want to use default profile (`default` in `~/.aws/credentials`), you don't have to specify here.
+Use this config section to specify shared credential file path and profile name. If you want to use default profile (`default` in `~/.aws/credentials`), you don't have to specify here. For example, you can specify the config like below:
+
+      <shared_credentials>
+        profile_name "your_profile_name"
+      </shared_credentials>
 
 #### profile_name
 Profile name of the credential file.
@@ -144,7 +148,11 @@ Profile name of the credential file.
 Path for the credential file.
 
 ### assume_role_credentials
-Use this config section for cross account access.
+Use this config section for cross account access. For example, you can specify the config like below:
+
+      <assume_role_credentials>
+        role_arn "your_role_arn_in_cross_account_to_assume"
+      </assume_role_credentials>
 
 #### role_arn
 IAM Role to be assumed with [AssumeRole][assume_role].
@@ -170,6 +178,9 @@ If your record contains a field whose string should be sent to Amazon Kinesis di
 ### log_truncate_max_size
 Integer, default 0. When emitting the log entry, the message will be truncated by this size to avoid infinite loop when the log is also sent to Kinesis. The value 0 (default) means no truncation.
 
+### reduce_max_size_error_message
+Boolean, default false. When each record exceeds the maximum size, the original record is put on its error message. If this parameter is turned on, the error message will contain only summarized data to prevent large traffic generated by the error message.
+
 ## Configuration: kinesis_streams
 Here are `kinesis_streams` specific configurations.
 
@@ -400,7 +411,7 @@ Minimum: 1
 Maximum (inclusive): 9223372036854775807
 
 #### record_max_buffered_time
-Maximum amount of itme (milliseconds) a record may spend being buffered before it gets sent. Records may be sent sooner than this depending on the other buffering limits.
+Maximum amount of time (milliseconds) a record may spend being buffered before it gets sent. Records may be sent sooner than this depending on the other buffering limits.
 
 This setting provides coarse ordering among records - any two records will be reordered by no more than twice this amount (assuming no failures and retries and equal network latency).
 

data/lib/fluent/plugin/kinesis_helper/class_methods.rb

@@ -92,6 +92,7 @@ module Fluent
         config_param :formatter,             :string,  default: 'json'
         config_param :data_key,              :string,  default: nil
         config_param :log_truncate_max_size, :integer, default: 0
+        config_param :reduce_max_size_error_message, :bool, default: false
       end
 
       def config_param_for_batch_request

data/lib/fluent/plugin/kinesis_helper/error.rb

@@ -24,8 +24,13 @@ module Fluent
     end
 
     class ExceedMaxRecordSizeError < SkipRecordError
-      def initialize(record)
-        super "Record size limit exceeded for #{record}"
+      def initialize(record, reduce_message)
+        if reduce_message == true
+          sampled_record = "#{record.slice(0,1024)}...#{record.slice(-1024,1024)}"
+          super "Record size limit exceeded for #{record.length}-bytes record: #{sampled_record}"
+        else
+          super "Record size limit exceeded for #{record}"
+        end
       end
     end
 

data/lib/fluent/plugin/kinesis_helper/format.rb

@@ -64,7 +64,7 @@ module Fluent
         converted = convert_format(tag, time, record)
         converted[:data] += "\n" if @append_new_line
         if converted[:data].size > MaxRecordSize
-          raise ExceedMaxRecordSizeError, converted[:data]
+          raise ExceedMaxRecordSizeError.new(converted[:data], @reduce_max_size_error_message)
         else
           converted
         end

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -23,6 +23,7 @@ module Fluent
     def write(chunk)
       records = convert_to_records(chunk)
       split_to_batches(records).each do |batch|
+        next unless batch.size > 0
         batch_request_with_retry(batch)
       end
       log.debug("Written #{records.size} records")

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 #  permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '1.1.3'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-02 00:00:00.000000000 Z
+date: 2017-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -325,7 +325,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis.