fluent-plugin-kafka 0.5.6 → 0.5.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 268cd322294a6f01fd118de3ef04c22997602d7c
4
- data.tar.gz: 570de398931cf778f147d08738eab20ad78b8a9c
3
+ metadata.gz: cb704a5814ca89bb71f2a9ead76631881a0c3f9e
4
+ data.tar.gz: 2b6eb6c80c5060a650f17e97bfbee67b478d0e79
5
5
  SHA512:
6
- metadata.gz: 8e81f53b95cb3ad41c9cb46779ba941021de37ae667f3d7d2be7d24b80e6f1603de95ba673e0bb86d9c4ec54a7b34243b5201cdfc35b7598b2eaa4807221b066
7
- data.tar.gz: f804bd9ea046af9dcc4e66081af471100fdcff044add13c0fd26bc3fe1c5241a6b758102b8a55649ef01c0fd0875d2bc1e2f0d87feca6f90b4e5959222e654a3
6
+ metadata.gz: b2e958ccbc9af8a9b0e4cffc87ad54054d5a579f584ec820a09eb02bcf9392c394938b3c7ad276a2a15b27f8d3d137c00b5f089566b2a03c5bcd0939528b6a68
7
+ data.tar.gz: 96ac9c4cf08269eef363677dc7dd38a316b54c82b5ca1504661770e210c7dd2c52bb1d7e1edbbeaa3a156b78e332bbe3170bc271ce200b2a46fa964beb59477b
data/ChangeLog CHANGED
@@ -1,3 +1,7 @@
1
+ Release 0.5.7 - 2017/07/13
2
+
3
+ * out_kafka_buffered: Add kafka_agg_max_messages parameter
4
+
1
5
  Release 0.5.6 - 2017/07/10
2
6
 
3
7
  * output: Add ActiveSupport notification support
data/README.md CHANGED
@@ -141,6 +141,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
141
141
  required_acks (integer) :default => -1
142
142
  ack_timeout (integer) :default => nil (Use default of ruby-kafka)
143
143
  compression_codec (gzip|snappy) :default => nil (No compression)
144
+ kafka_agg_max_bytes (integer) :default => 4096
145
+ kafka_agg_max_messages (integer) :default => nil (No limit)
144
146
  max_send_limit_bytes (integer) :default => nil (No drop)
145
147
  discard_kafka_delivery_failed (bool) :default => false (No discard)
146
148
  </match>
@@ -157,6 +159,8 @@ Supports following ruby-kafka's producer options.
157
159
  - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
158
160
  - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
159
161
  - compression_codec - default: nil - The codec the producer uses to compress messages.
162
+ - kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
163
+ - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
160
164
  - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
161
165
  - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
162
166
 
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.5.6'
15
+ gem.version = '0.5.7'
16
16
  gem.required_ruby_version = ">= 2.1.0"
17
17
 
18
18
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -48,6 +48,7 @@ Set true to remove topic name key from data
48
48
  DESC
49
49
 
50
50
  config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
51
+ config_param :kafka_agg_max_messages, :integer, :default => nil
51
52
  config_param :get_kafka_client_log, :bool, :default => false
52
53
 
53
54
  # ruby-kafka producer options
@@ -287,8 +288,8 @@ DESC
287
288
  next
288
289
  end
289
290
 
290
- if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
291
- log.debug { "#{messages} messages send because reaches kafka_agg_max_bytes" }
291
+ if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes) or (@kafka_agg_max_messages && messages >= @kafka_agg_max_messages)
292
+ log.debug { "#{messages} messages send because reaches the limit of batch transmission." }
292
293
  deliver_messages(producer, tag)
293
294
  messages = 0
294
295
  messages_bytes = 0
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.6
4
+ version: 0.5.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-07-10 00:00:00.000000000 Z
12
+ date: 2017-07-12 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd