fluent-plugin-kafka 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -0
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/out_kafka.rb +12 -3
- data/lib/fluent/plugin/out_kafka_buffered.rb +12 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2d81baf9816d05b5cc5f5c982aafe722781d978
|
4
|
+
data.tar.gz: cfb1477ba8cd4961424881c173cf0d4dffab999d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 15b62cb6289281bebf2938172b41fe0a8f7080d6afebfebf672e45a10a1c31779db3ed9b739a28db57300ed5323abd814ae69720e0afc0d3a301ffebfce66e8b
|
7
|
+
data.tar.gz: 70deeef4310ef4ad2674ba21fb2f981147bc0784810d88c14ab32efe837f7b1f7fc41f6bb540223d65bc1df0cae599440ae05eaaae7ded3185c6865a727fedbb
|
data/README.md
CHANGED
@@ -116,6 +116,8 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
|
|
116
116
|
output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
|
117
117
|
output_include_tag (bool) :default => false
|
118
118
|
output_include_time (bool) :default => false
|
119
|
+
exclude_topic_key (bool) :default => false
|
120
|
+
exclude_partition_key (bool) :default => false
|
119
121
|
|
120
122
|
# ruby-kafka producer options
|
121
123
|
max_send_retries (integer) :default => 1
|
@@ -173,6 +175,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
|
|
173
175
|
output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
|
174
176
|
output_include_tag (bool) :default => false
|
175
177
|
output_include_time (bool) :default => false
|
178
|
+
exclude_topic_key (bool) :default => false
|
179
|
+
exclude_partition_key (bool) :default => false
|
176
180
|
get_kafka_client_log (bool) :default => false
|
177
181
|
|
178
182
|
# See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.3.
|
15
|
+
gem.version = '0.3.4'
|
16
16
|
gem.required_ruby_version = ">= 2.1.0"
|
17
17
|
|
18
18
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -22,9 +22,17 @@ DESC
|
|
22
22
|
:desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
|
23
23
|
config_param :output_include_tag, :bool, :default => false
|
24
24
|
config_param :output_include_time, :bool, :default => false
|
25
|
+
config_param :exclude_partition_key, :bool, :default => false,
|
26
|
+
:desc => <<-DESC
|
27
|
+
Set true to remove partition key from data
|
28
|
+
DESC
|
29
|
+
config_param :exclude_topic_key, :bool, :default => false,
|
30
|
+
:desc => <<-DESC
|
31
|
+
Set true to remove topic name key from data
|
32
|
+
DESC
|
25
33
|
|
26
34
|
# ruby-kafka producer options
|
27
|
-
config_param :max_send_retries, :integer, :default =>
|
35
|
+
config_param :max_send_retries, :integer, :default => 2,
|
28
36
|
:desc => "Number of times to retry sending of messages to a leader."
|
29
37
|
config_param :required_acks, :integer, :default => -1,
|
30
38
|
:desc => "The number of acks required per request."
|
@@ -157,8 +165,9 @@ DESC
|
|
157
165
|
end
|
158
166
|
end
|
159
167
|
record['tag'] = tag if @output_include_tag
|
160
|
-
topic = record['topic'] || @default_topic || tag
|
161
|
-
partition_key = record['partition_key'] || @default_partition_key
|
168
|
+
topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag
|
169
|
+
partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
|
170
|
+
|
162
171
|
value = @formatter_proc.call(tag, time, record)
|
163
172
|
|
164
173
|
log.on_trace { log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") }
|
@@ -28,11 +28,20 @@ Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
|
28
28
|
DESC
|
29
29
|
config_param :output_include_tag, :bool, :default => false
|
30
30
|
config_param :output_include_time, :bool, :default => false
|
31
|
+
config_param :exclude_partition_key, :bool, :default => false,
|
32
|
+
:desc => <<-DESC
|
33
|
+
Set true to remove partition key from data
|
34
|
+
DESC
|
35
|
+
config_param :exclude_topic_key, :bool, :default => false,
|
36
|
+
:desc => <<-DESC
|
37
|
+
Set true to remove topic name key from data
|
38
|
+
DESC
|
39
|
+
|
31
40
|
config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
|
32
41
|
config_param :get_kafka_client_log, :bool, :default => false
|
33
42
|
|
34
43
|
# ruby-kafka producer options
|
35
|
-
config_param :max_send_retries, :integer, :default =>
|
44
|
+
config_param :max_send_retries, :integer, :default => 2,
|
36
45
|
:desc => "Number of times to retry sending of messages to a leader."
|
37
46
|
config_param :required_acks, :integer, :default => -1,
|
38
47
|
:desc => "The number of acks required per request."
|
@@ -213,8 +222,8 @@ DESC
|
|
213
222
|
end
|
214
223
|
|
215
224
|
record['tag'] = tag if @output_include_tag
|
216
|
-
topic = record['topic'.freeze] || def_topic
|
217
|
-
partition_key = record['partition_key'.freeze] || @default_partition_key
|
225
|
+
topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
|
226
|
+
partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
|
218
227
|
|
219
228
|
records_by_topic[topic] ||= 0
|
220
229
|
bytes_by_topic[topic] ||= 0
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-10-
|
12
|
+
date: 2016-10-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|