fluent-plugin-kafka 0.3.2 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 512f9e6a1ab34f3206a79904ff3134da1ee6de54
4
- data.tar.gz: 27bcf99b9a4eac5ed2cf15694a249db4f3db6ffb
3
+ metadata.gz: 0dc35a4ee0bdd32fef9268ffb4a89a163fd7d78a
4
+ data.tar.gz: 64db45e59ff24473f1adabde25bc2a65f237fbb6
5
5
  SHA512:
6
- metadata.gz: 0d926b09af1ed8611d85a40eb353aeda21b39f862c85a4131ada9e9fcfa9d057157fb107a56808af905c89c669c6a452b03eef508e665ae3957f7599c25c19af
7
- data.tar.gz: 60d029566a17521e19dd93e0e6d2cefd35990a8b14d89202c42ba6cf6512608f78dced06e23d0e1386e95d206173ad78a77c65dba081f1c3e5adc150f74c07d4
6
+ metadata.gz: 58fdebb9585c65e30b290d3f72707046942873e86220fdc4207c6d3194faca3cf015c487d71df2f61c491a9089210b35c2015a4d0ba639f5acfa30b1250cc46f
7
+ data.tar.gz: 6c331f30a9c19d58f66b916b7ef01f85a999121710e34eb6352db0f4a43b08af436ca017fd56e09b5a9a6ef825ffd92cadbb900d99c88a3972aa6fe7dcb7a13a
data/README.md CHANGED
@@ -173,6 +173,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
173
173
  output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
174
174
  output_include_tag (bool) :default => false
175
175
  output_include_time (bool) :default => false
176
+ get_kafka_client_log (bool) :default => false
176
177
 
177
178
  # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
178
179
 
@@ -183,6 +184,12 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
183
184
  compression_codec (gzip|snappy) :default => nil (No compression)
184
185
  </match>
185
186
 
187
+ `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
188
+
189
+ ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
190
+ In this case, `get_kafka_client_log` is useful for identifying the error cause.
191
+ ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
192
+
186
193
  Supports following ruby-kafka's producer options.
187
194
 
188
195
  - max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.3.2'
15
+ gem.version = '0.3.3'
16
16
  gem.required_ruby_version = ">= 2.1.0"
17
17
 
18
18
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -18,6 +18,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
18
18
  :desc => "Tag prefix (Optional)"
19
19
  config_param :add_suffix, :string, :default => nil,
20
20
  :desc => "Tag suffix (Optional)"
21
+ config_param :retry_emit_limit, :integer, :default => nil,
22
+ :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
21
23
 
22
24
  # Kafka consumer options
23
25
  config_param :max_wait_time, :integer, :default => nil,
@@ -35,6 +37,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
35
37
 
36
38
  include Fluent::KafkaPluginUtil::SSLSettings
37
39
 
40
+ class ForShutdown < StandardError
41
+ end
42
+
38
43
  unless method_defined?(:router)
39
44
  define_method("router") { Fluent::Engine }
40
45
  end
@@ -146,9 +151,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
146
151
  }
147
152
 
148
153
  unless es.empty?
149
- router.emit_stream(tag, es)
154
+ emit_events(tag, es)
150
155
  end
151
156
  }
157
+ rescue ForShutdown
152
158
  rescue => e
153
159
  log.error "unexpected error during consuming events from kafka. Re-fetch events.", :error => e.to_s
154
160
  log.error_backtrace
@@ -158,4 +164,26 @@ class Fluent::KafkaGroupInput < Fluent::Input
158
164
  log.error "unexpected error during consumer object access", :error => e.to_s
159
165
  log.error_backtrace
160
166
  end
167
+
168
+ def emit_events(tag, es)
169
+ retries = 0
170
+ begin
171
+ router.emit_stream(tag, es)
172
+ rescue Fluent::BufferQueueLimitError
173
+ raise ForShutdown if @consumer.nil?
174
+
175
+ if @retry_emit_limit.nil?
176
+ sleep 1
177
+ retry
178
+ end
179
+
180
+ if retries < @retry_emit_limit
181
+ retries += 1
182
+ sleep 1
183
+ retry
184
+ else
185
+ raise RuntimeError, "Exceeds retry_emit_limit"
186
+ end
187
+ end
188
+ end
161
189
  end
@@ -29,6 +29,7 @@ DESC
29
29
  config_param :output_include_tag, :bool, :default => false
30
30
  config_param :output_include_time, :bool, :default => false
31
31
  config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
32
+ config_param :get_kafka_client_log, :bool, :default => false
32
33
 
33
34
  # ruby-kafka producer options
34
35
  config_param :max_send_retries, :integer, :default => 1,
@@ -78,7 +79,8 @@ DESC
78
79
  end
79
80
  begin
80
81
  if @seed_brokers.length > 0
81
- @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
82
+ logger = @get_kafka_client_log ? log : nil
83
+ @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
82
84
  ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
83
85
  log.info "initialized kafka producer: #{@client_id}"
84
86
  else
@@ -196,25 +198,34 @@ DESC
196
198
  bytes_by_topic = {}
197
199
  messages = 0
198
200
  messages_bytes = 0
201
+ record_buf = nil
202
+ record_buf_bytes = nil
203
+
199
204
  begin
200
205
  chunk.msgpack_each { |time, record|
201
- if @output_include_time
202
- if @time_format
203
- record['time'.freeze] = Time.at(time).strftime(@time_format)
204
- else
205
- record['time'.freeze] = time
206
+ begin
207
+ if @output_include_time
208
+ if @time_format
209
+ record['time'.freeze] = Time.at(time).strftime(@time_format)
210
+ else
211
+ record['time'.freeze] = time
212
+ end
206
213
  end
207
- end
208
214
 
209
- record['tag'] = tag if @output_include_tag
210
- topic = record['topic'.freeze] || def_topic
211
- partition_key = record['partition_key'.freeze] || @default_partition_key
215
+ record['tag'] = tag if @output_include_tag
216
+ topic = record['topic'.freeze] || def_topic
217
+ partition_key = record['partition_key'.freeze] || @default_partition_key
212
218
 
213
- records_by_topic[topic] ||= 0
214
- bytes_by_topic[topic] ||= 0
219
+ records_by_topic[topic] ||= 0
220
+ bytes_by_topic[topic] ||= 0
221
+
222
+ record_buf = @formatter_proc.call(tag, time, record)
223
+ record_buf_bytes = record_buf.bytesize
224
+ rescue StandardError => e
225
+ log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
226
+ next
227
+ end
215
228
 
216
- record_buf = @formatter_proc.call(tag, time, record)
217
- record_buf_bytes = record_buf.bytesize
218
229
  if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
219
230
  log.on_trace { log.trace("#{messages} messages send.") }
220
231
  producer.deliver_messages
@@ -237,6 +248,7 @@ DESC
237
248
  end
238
249
  rescue Exception => e
239
250
  log.warn "Send exception occurred: #{e}"
251
+ log.warn "Exception Backtrace : #{e.backtrace.join("\n")}"
240
252
  # For safety, refresh client and its producers
241
253
  shutdown_producers
242
254
  refresh_client(false)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.3.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-10-06 00:00:00.000000000 Z
12
+ date: 2016-10-17 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd