fluent-plugin-kafka 0.3.3 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0dc35a4ee0bdd32fef9268ffb4a89a163fd7d78a
4
- data.tar.gz: 64db45e59ff24473f1adabde25bc2a65f237fbb6
3
+ metadata.gz: e2d81baf9816d05b5cc5f5c982aafe722781d978
4
+ data.tar.gz: cfb1477ba8cd4961424881c173cf0d4dffab999d
5
5
  SHA512:
6
- metadata.gz: 58fdebb9585c65e30b290d3f72707046942873e86220fdc4207c6d3194faca3cf015c487d71df2f61c491a9089210b35c2015a4d0ba639f5acfa30b1250cc46f
7
- data.tar.gz: 6c331f30a9c19d58f66b916b7ef01f85a999121710e34eb6352db0f4a43b08af436ca017fd56e09b5a9a6ef825ffd92cadbb900d99c88a3972aa6fe7dcb7a13a
6
+ metadata.gz: 15b62cb6289281bebf2938172b41fe0a8f7080d6afebfebf672e45a10a1c31779db3ed9b739a28db57300ed5323abd814ae69720e0afc0d3a301ffebfce66e8b
7
+ data.tar.gz: 70deeef4310ef4ad2674ba21fb2f981147bc0784810d88c14ab32efe837f7b1f7fc41f6bb540223d65bc1df0cae599440ae05eaaae7ded3185c6865a727fedbb
data/README.md CHANGED
@@ -116,6 +116,8 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
116
116
  output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
117
117
  output_include_tag (bool) :default => false
118
118
  output_include_time (bool) :default => false
119
+ exclude_topic_key (bool) :default => false
120
+ exclude_partition_key (bool) :default => false
119
121
 
120
122
  # ruby-kafka producer options
121
123
  max_send_retries (integer) :default => 1
@@ -173,6 +175,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
173
175
  output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
174
176
  output_include_tag (bool) :default => false
175
177
  output_include_time (bool) :default => false
178
+ exclude_topic_key (bool) :default => false
179
+ exclude_partition_key (bool) :default => false
176
180
  get_kafka_client_log (bool) :default => false
177
181
 
178
182
  # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.3.3'
15
+ gem.version = '0.3.4'
16
16
  gem.required_ruby_version = ">= 2.1.0"
17
17
 
18
18
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -22,9 +22,17 @@ DESC
22
22
  :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
23
23
  config_param :output_include_tag, :bool, :default => false
24
24
  config_param :output_include_time, :bool, :default => false
25
+ config_param :exclude_partition_key, :bool, :default => false,
26
+ :desc => <<-DESC
27
+ Set true to remove partition key from data
28
+ DESC
29
+ config_param :exclude_topic_key, :bool, :default => false,
30
+ :desc => <<-DESC
31
+ Set true to remove topic name key from data
32
+ DESC
25
33
 
26
34
  # ruby-kafka producer options
27
- config_param :max_send_retries, :integer, :default => 1,
35
+ config_param :max_send_retries, :integer, :default => 2,
28
36
  :desc => "Number of times to retry sending of messages to a leader."
29
37
  config_param :required_acks, :integer, :default => -1,
30
38
  :desc => "The number of acks required per request."
@@ -157,8 +165,9 @@ DESC
157
165
  end
158
166
  end
159
167
  record['tag'] = tag if @output_include_tag
160
- topic = record['topic'] || @default_topic || tag
161
- partition_key = record['partition_key'] || @default_partition_key
168
+ topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag
169
+ partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
170
+
162
171
  value = @formatter_proc.call(tag, time, record)
163
172
 
164
173
  log.on_trace { log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") }
@@ -28,11 +28,20 @@ Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
28
28
  DESC
29
29
  config_param :output_include_tag, :bool, :default => false
30
30
  config_param :output_include_time, :bool, :default => false
31
+ config_param :exclude_partition_key, :bool, :default => false,
32
+ :desc => <<-DESC
33
+ Set true to remove partition key from data
34
+ DESC
35
+ config_param :exclude_topic_key, :bool, :default => false,
36
+ :desc => <<-DESC
37
+ Set true to remove topic name key from data
38
+ DESC
39
+
31
40
  config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
32
41
  config_param :get_kafka_client_log, :bool, :default => false
33
42
 
34
43
  # ruby-kafka producer options
35
- config_param :max_send_retries, :integer, :default => 1,
44
+ config_param :max_send_retries, :integer, :default => 2,
36
45
  :desc => "Number of times to retry sending of messages to a leader."
37
46
  config_param :required_acks, :integer, :default => -1,
38
47
  :desc => "The number of acks required per request."
@@ -213,8 +222,8 @@ DESC
213
222
  end
214
223
 
215
224
  record['tag'] = tag if @output_include_tag
216
- topic = record['topic'.freeze] || def_topic
217
- partition_key = record['partition_key'.freeze] || @default_partition_key
225
+ topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
226
+ partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
218
227
 
219
228
  records_by_topic[topic] ||= 0
220
229
  bytes_by_topic[topic] ||= 0
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.3
4
+ version: 0.3.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-10-17 00:00:00.000000000 Z
12
+ date: 2016-10-20 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd