fluent-plugin-kinesis-modified 3.1.4 → 3.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e5eb2e11af14feafefd700f1cdccbf3e2db10d83
-  data.tar.gz: c439a086a1cd5fdcfab0134c70737c80f4fcc833
+  metadata.gz: 7765915d41361a75eb09b522c3fa44d5dbf9b8a3
+  data.tar.gz: f282294b383621d02a469b1c12f5785d82afce3f
 SHA512:
-  metadata.gz: f2a8c698e569f3aa524c1b11a9f5b9140340532d06bc94b051474920993b4922f16c124bfb29f1f102e83764bd0d3f880720983ddb4ae6b3806056fe33000bed
-  data.tar.gz: 7c7f1c908d027e9e787e65f5c6c9e921aa76382ad14d2ef36ae86a4e2ae09564af9ed07627cd57b6fa6497182741b50bf54866623f6b61730b82b09000ff71dd
+  metadata.gz: 8d91bc03c04ae38fae648a33cac55fd609b18ea55d79959a72199bbd98073198103a1f85025a9e790f325c67b424294dd494558036cc5429220a6188b747cd8b
+  data.tar.gz: 4a5f7325bd584c9e2e9d585d18fb60a69b18ba5f3c9ad8039f81d6d2fca37f669b5bd0ebac4e7aeaf1c365529fde94c5506189e7fb499739be1f81cb4441e6ad

data/lib/fluent/plugin/kinesis.rb

@@ -72,12 +72,14 @@ module Fluent
 
       config_param :debug, :bool, default: false
       config_param :max_records_per_call, :integer, default: 128
+      config_param :max_request_size, :integer, default: 4*1024
 
       helpers :formatter, :inject
 
       def configure(conf)
         super
         @data_formatter = data_formatter_create(conf)
+        @max_request_size *= 1024
       end
 
       def multi_workers_ready?
@@ -151,20 +153,24 @@ module Fluent
         chunk.open do |io|
           records = msgpack_unpacker(io).to_enum
           batches = []
-          sizes = []
+          batches_size = 0
           split_to_batches(records) do |batch, size|
-            batches << batch
-            sizes << size            
-          end
-          (0..batches.size).step(@max_records_per_call) do |i|
-            batches_slice = batches[i..i+@max_records_per_call-1]
-            size = sizes[i..i+@max_records_per_call-1].inject(:+)
-            batches_size = batches_slice.map(&:size).inject(:+)
-            if not size.nil?
-              log.debug(sprintf "Write chunk %s / %3d batches / %3d records / %4d KB", unique_id, batches_slice.size, batches_size, size/1024)
-              batch_request_with_retry(batches_slice, &block)
+            if (batches.size+1 > @max_records_per_call or batches_size+size > @max_request_size) and batches.size > 0
+              records_number = batches.map(&:size).inject(:+)
+              log.debug(sprintf "Write chunk %s / %3d batches / %3d records / %4d KB", unique_id, batches.size, records_number, batches_size/1024)
+              batch_request_with_retry(batches, &block)
               log.debug("Finish writing chunk")
+              batches = []
+              batches_size = 0
             end
+            batches << batch
+            batches_size += size            
+          end
+          if batches.size > 0
+            records_number = batches.map(&:size).inject(:+)
+            log.debug(sprintf "Write chunk %s / %3d batches / %3d records / %4d KB", unique_id, batches.size, records_number, batches_size/1024)
+            batch_request_with_retry(batches, &block)
+            log.debug("Finish writing chunk")
           end
         end
       end

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.1.4'
+  VERSION = '3.1.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis-modified
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.1.5
 platform: ruby
 authors:
 - iHandy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-13 00:00:00.000000000 Z
+date: 2019-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -305,7 +305,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis. Forked from fluent-plugin-kinesis