fluent-plugin-kinesis-modified 3.1.4 → 3.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fluent/plugin/kinesis.rb +17 -11
- data/lib/fluent_plugin_kinesis/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7765915d41361a75eb09b522c3fa44d5dbf9b8a3
|
4
|
+
data.tar.gz: f282294b383621d02a469b1c12f5785d82afce3f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d91bc03c04ae38fae648a33cac55fd609b18ea55d79959a72199bbd98073198103a1f85025a9e790f325c67b424294dd494558036cc5429220a6188b747cd8b
|
7
|
+
data.tar.gz: 4a5f7325bd584c9e2e9d585d18fb60a69b18ba5f3c9ad8039f81d6d2fca37f669b5bd0ebac4e7aeaf1c365529fde94c5506189e7fb499739be1f81cb4441e6ad
|
@@ -72,12 +72,14 @@ module Fluent
|
|
72
72
|
|
73
73
|
config_param :debug, :bool, default: false
|
74
74
|
config_param :max_records_per_call, :integer, default: 128
|
75
|
+
config_param :max_request_size, :integer, default: 4*1024
|
75
76
|
|
76
77
|
helpers :formatter, :inject
|
77
78
|
|
78
79
|
def configure(conf)
|
79
80
|
super
|
80
81
|
@data_formatter = data_formatter_create(conf)
|
82
|
+
@max_request_size *= 1024
|
81
83
|
end
|
82
84
|
|
83
85
|
def multi_workers_ready?
|
@@ -151,20 +153,24 @@ module Fluent
|
|
151
153
|
chunk.open do |io|
|
152
154
|
records = msgpack_unpacker(io).to_enum
|
153
155
|
batches = []
|
154
|
-
|
156
|
+
batches_size = 0
|
155
157
|
split_to_batches(records) do |batch, size|
|
156
|
-
batches
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
batches_slice = batches[i..i+@max_records_per_call-1]
|
161
|
-
size = sizes[i..i+@max_records_per_call-1].inject(:+)
|
162
|
-
batches_size = batches_slice.map(&:size).inject(:+)
|
163
|
-
if not size.nil?
|
164
|
-
log.debug(sprintf "Write chunk %s / %3d batches / %3d records / %4d KB", unique_id, batches_slice.size, batches_size, size/1024)
|
165
|
-
batch_request_with_retry(batches_slice, &block)
|
158
|
+
if (batches.size+1 > @max_records_per_call or batches_size+size > @max_request_size) and batches.size > 0
|
159
|
+
records_number = batches.map(&:size).inject(:+)
|
160
|
+
log.debug(sprintf "Write chunk %s / %3d batches / %3d records / %4d KB", unique_id, batches.size, records_number, batches_size/1024)
|
161
|
+
batch_request_with_retry(batches, &block)
|
166
162
|
log.debug("Finish writing chunk")
|
163
|
+
batches = []
|
164
|
+
batches_size = 0
|
167
165
|
end
|
166
|
+
batches << batch
|
167
|
+
batches_size += size
|
168
|
+
end
|
169
|
+
if batches.size > 0
|
170
|
+
records_number = batches.map(&:size).inject(:+)
|
171
|
+
log.debug(sprintf "Write chunk %s / %3d batches / %3d records / %4d KB", unique_id, batches.size, records_number, batches_size/1024)
|
172
|
+
batch_request_with_retry(batches, &block)
|
173
|
+
log.debug("Finish writing chunk")
|
168
174
|
end
|
169
175
|
end
|
170
176
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kinesis-modified
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- iHandy
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -305,7 +305,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
305
305
|
version: '0'
|
306
306
|
requirements: []
|
307
307
|
rubyforge_project:
|
308
|
-
rubygems_version: 2.5.2
|
308
|
+
rubygems_version: 2.5.2.3
|
309
309
|
signing_key:
|
310
310
|
specification_version: 4
|
311
311
|
summary: Fluentd output plugin that sends events to Amazon Kinesis. Forked from fluent-plugin-kinesis
|