fluent-plugin-kafka 0.15.3 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 730f14a0d24b98ac7df9683bc0836f826ff80def0ee61c833bc4c1e61bf435ac
4
- data.tar.gz: 0a647554fb96f88ae75c8b872399545bfdb6b26d12247f3715c56dda2e628d57
3
+ metadata.gz: 7fcaf3e8fbb836ab8db3fa21003a713f9048de076097e2c95a9f30e5d1b05c08
4
+ data.tar.gz: 8437b2c9401238d811973422a65d2c9ee34bd8afc513f32412a22f93e03204a0
5
5
  SHA512:
6
- metadata.gz: 802bef06a0cb0703e0e3c1d8c9581a96007b23a6b574d2c64f3bca362f6d4c531cbc4c9c17c78162705d655fc2e864af16182772a1db0ce2150c0e853b4f55ef
7
- data.tar.gz: 0d3695bc1289b75cee1071fcf8e697250de9f0177b3facd69dd62e931a7b705fd93b24f886e683586d01f9daf9888d487ff6058da791d3779d09c076af854dd8
6
+ metadata.gz: 6d407e6f12dbafc6f5fad59ac6cbae4b13fa5f0a212cf82dba604f871afc286899290972b3afca5d1031888ee2f141c12d5d2e3d6b7324c7246317029149c491
7
+ data.tar.gz: b278441842361cc53836fce087e87e6d2d800bccbe9b307c2f0b39d3c4c357aae96275d2bfb808ab2455ebce692948d1241f4f991a5036473da1d31da699832b
@@ -0,0 +1,26 @@
1
+ name: linux
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0' ]
12
+ os:
13
+ - ubuntu-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
data/ChangeLog CHANGED
@@ -1,3 +1,8 @@
1
+ Release 0.16.0 - 2021/01/25
2
+
3
+ * input: Add `tag_source` and `record_tag_key` parameters for using record field as tag
4
+ * in_kafka_group: Use NumericParser for floating point
5
+
1
6
  Release 0.15.3 - 2020/12/08
2
7
 
3
8
  * in_kafka: Fix `record_time_key` parameter not working
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
13
13
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
14
  gem.name = "fluent-plugin-kafka"
15
15
  gem.require_paths = ["lib"]
16
- gem.version = '0.15.3'
16
+ gem.version = '0.16.0'
17
17
  gem.required_ruby_version = ">= 2.1.0"
18
18
 
19
19
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -21,4 +21,5 @@ Gem::Specification.new do |gem|
21
21
  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
22
22
  gem.add_development_dependency "rake", ">= 0.9.2"
23
23
  gem.add_development_dependency "test-unit", ">= 3.0.8"
24
+ gem.add_development_dependency "webrick"
24
25
  end
@@ -31,6 +31,10 @@ class Fluent::KafkaInput < Fluent::Input
31
31
  config_param :add_suffix, :string, :default => nil,
32
32
  :desc => "tag suffix"
33
33
  config_param :add_offset_in_record, :bool, :default => false
34
+ config_param :tag_source, :enum, :list => [:topic, :record], :default => :topic,
35
+ :desc => "Source for the fluentd event tag"
36
+ config_param :record_tag_key, :string, :default => 'tag',
37
+ :desc => "Tag field when tag_source is 'record'"
34
38
 
35
39
  config_param :offset_zookeeper, :string, :default => nil
36
40
  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
@@ -225,6 +229,8 @@ class Fluent::KafkaInput < Fluent::Input
225
229
  @kafka_message_key,
226
230
  @time_source,
227
231
  @record_time_key,
232
+ @tag_source,
233
+ @record_tag_key,
228
234
  opt)
229
235
  }
230
236
  @topic_watchers.each {|tw|
@@ -249,7 +255,7 @@ class Fluent::KafkaInput < Fluent::Input
249
255
  end
250
256
 
251
257
  class TopicWatcher < Coolio::TimerWatcher
252
- def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, record_time_key, options={})
258
+ def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, record_time_key, tag_source, record_tag_key, options={})
253
259
  @topic_entry = topic_entry
254
260
  @kafka = kafka
255
261
  @callback = method(:consume)
@@ -262,6 +268,8 @@ class Fluent::KafkaInput < Fluent::Input
262
268
  @kafka_message_key = kafka_message_key
263
269
  @time_source = time_source
264
270
  @record_time_key = record_time_key
271
+ @tag_source = tag_source
272
+ @record_tag_key = record_tag_key
265
273
 
266
274
  @next_offset = @topic_entry.offset
267
275
  if @topic_entry.offset == -1 && offset_manager
@@ -298,6 +306,11 @@ class Fluent::KafkaInput < Fluent::Input
298
306
  messages.each { |msg|
299
307
  begin
300
308
  record = @parser.call(msg, @topic_entry)
309
+ if @tag_source == :record
310
+ tag = record[@record_tag_key]
311
+ tag = @add_prefix + "." + tag if @add_prefix
312
+ tag = tag + "." + @add_suffix if @add_suffix
313
+ end
301
314
  case @time_source
302
315
  when :kafka
303
316
  record_time = Fluent::EventTime.from_time(msg.create_time)
@@ -36,6 +36,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
36
36
  config_param :get_kafka_client_log, :bool, :default => false
37
37
  config_param :time_format, :string, :default => nil,
38
38
  :desc => "Time format to be used to parse 'time' field."
39
+ config_param :tag_source, :enum, :list => [:topic, :record], :default => :topic,
40
+ :desc => "Source for the fluentd event tag"
41
+ config_param :record_tag_key, :string, :default => 'tag',
42
+ :desc => "Tag field when tag_source is 'record'"
39
43
  config_param :kafka_message_key, :string, :default => nil,
40
44
  :desc => "Set kafka's message key to this field"
41
45
  config_param :connect_timeout, :integer, :default => nil,
@@ -138,6 +142,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
138
142
  @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
139
143
  end
140
144
  end
145
+
146
+ if @time_source == :record && defined?(Fluent::NumericTimeParser)
147
+ @float_numeric_parse = Fluent::NumericTimeParser.new(:float)
148
+ end
141
149
  end
142
150
 
143
151
  def setup_parser(conf)
@@ -244,49 +252,104 @@ class Fluent::KafkaGroupInput < Fluent::Input
244
252
  end
245
253
  end
246
254
 
255
+ def process_batch_with_record_tag(batch)
256
+ es = {}
257
+ batch.messages.each { |msg|
258
+ begin
259
+ record = @parser_proc.call(msg)
260
+ tag = record[@record_tag_key]
261
+ tag = @add_prefix + "." + tag if @add_prefix
262
+ tag = tag + "." + @add_suffix if @add_suffix
263
+ es[tag] ||= Fluent::MultiEventStream.new
264
+ case @time_source
265
+ when :kafka
266
+ record_time = Fluent::EventTime.from_time(msg.create_time)
267
+ when :now
268
+ record_time = Fluent::Engine.now
269
+ when :record
270
+ if @time_format
271
+ record_time = @time_parser.parse(record[@record_time_key].to_s)
272
+ else
273
+ record_time = record[@record_time_key]
274
+ end
275
+ else
276
+ log.fatal "BUG: invalid time_source: #{@time_source}"
277
+ end
278
+ if @kafka_message_key
279
+ record[@kafka_message_key] = msg.key
280
+ end
281
+ if @add_headers
282
+ msg.headers.each_pair { |k, v|
283
+ record[k] = v
284
+ }
285
+ end
286
+ es[tag].add(record_time, record)
287
+ rescue => e
288
+ log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
289
+ log.debug_backtrace
290
+ end
291
+ }
292
+
293
+ unless es.empty?
294
+ es.each { |tag,es|
295
+ emit_events(tag, es)
296
+ }
297
+ end
298
+ end
299
+
300
+ def process_batch(batch)
301
+ es = Fluent::MultiEventStream.new
302
+ tag = batch.topic
303
+ tag = @add_prefix + "." + tag if @add_prefix
304
+ tag = tag + "." + @add_suffix if @add_suffix
305
+
306
+ batch.messages.each { |msg|
307
+ begin
308
+ record = @parser_proc.call(msg)
309
+ case @time_source
310
+ when :kafka
311
+ record_time = Fluent::EventTime.from_time(msg.create_time)
312
+ when :now
313
+ record_time = Fluent::Engine.now
314
+ when :record
315
+ record_time = record[@record_time_key]
316
+
317
+ if @time_format
318
+ record_time = @time_parser.parse(record_time.to_s)
319
+ elsif record_time.is_a?(Float) && @float_numeric_parse
320
+ record_time = @float_numeric_parse.parse(record_time)
321
+ end
322
+ else
323
+ log.fatal "BUG: invalid time_source: #{@time_source}"
324
+ end
325
+ if @kafka_message_key
326
+ record[@kafka_message_key] = msg.key
327
+ end
328
+ if @add_headers
329
+ msg.headers.each_pair { |k, v|
330
+ record[k] = v
331
+ }
332
+ end
333
+ es.add(record_time, record)
334
+ rescue => e
335
+ log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
336
+ log.debug_backtrace
337
+ end
338
+ }
339
+
340
+ unless es.empty?
341
+ emit_events(tag, es)
342
+ end
343
+ end
344
+
247
345
  def run
248
346
  while @consumer
249
347
  begin
250
348
  @consumer.each_batch(@fetch_opts) { |batch|
251
- es = Fluent::MultiEventStream.new
252
- tag = batch.topic
253
- tag = @add_prefix + "." + tag if @add_prefix
254
- tag = tag + "." + @add_suffix if @add_suffix
255
-
256
- batch.messages.each { |msg|
257
- begin
258
- record = @parser_proc.call(msg)
259
- case @time_source
260
- when :kafka
261
- record_time = Fluent::EventTime.from_time(msg.create_time)
262
- when :now
263
- record_time = Fluent::Engine.now
264
- when :record
265
- if @time_format
266
- record_time = @time_parser.parse(record[@record_time_key].to_s)
267
- else
268
- record_time = record[@record_time_key]
269
- end
270
- else
271
- log.fatal "BUG: invalid time_source: #{@time_source}"
272
- end
273
- if @kafka_message_key
274
- record[@kafka_message_key] = msg.key
275
- end
276
- if @add_headers
277
- msg.headers.each_pair { |k, v|
278
- record[k] = v
279
- }
280
- end
281
- es.add(record_time, record)
282
- rescue => e
283
- log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
284
- log.debug_backtrace
285
- end
286
- }
287
-
288
- unless es.empty?
289
- emit_events(tag, es)
349
+ if @tag_source == :record
350
+ process_batch_with_record_tag(batch)
351
+ else
352
+ process_batch(batch)
290
353
  end
291
354
  }
292
355
  rescue ForShutdown
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.3
4
+ version: 0.16.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-12-08 00:00:00.000000000 Z
12
+ date: 2021-01-25 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -93,6 +93,20 @@ dependencies:
93
93
  - - ">="
94
94
  - !ruby/object:Gem::Version
95
95
  version: 3.0.8
96
+ - !ruby/object:Gem::Dependency
97
+ name: webrick
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ type: :development
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
96
110
  description: Fluentd plugin for Apache Kafka > 0.8
97
111
  email:
98
112
  - togachiro@gmail.com
@@ -101,6 +115,7 @@ executables: []
101
115
  extensions: []
102
116
  extra_rdoc_files: []
103
117
  files:
118
+ - ".github/workflows/linux.yml"
104
119
  - ".gitignore"
105
120
  - ".travis.yml"
106
121
  - ChangeLog