fluent-plugin-kafka 0.15.3 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +26 -0
- data/ChangeLog +5 -0
- data/fluent-plugin-kafka.gemspec +2 -1
- data/lib/fluent/plugin/in_kafka.rb +14 -1
- data/lib/fluent/plugin/in_kafka_group.rb +102 -39
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7fcaf3e8fbb836ab8db3fa21003a713f9048de076097e2c95a9f30e5d1b05c08
|
4
|
+
data.tar.gz: 8437b2c9401238d811973422a65d2c9ee34bd8afc513f32412a22f93e03204a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d407e6f12dbafc6f5fad59ac6cbae4b13fa5f0a212cf82dba604f871afc286899290972b3afca5d1031888ee2f141c12d5d2e3d6b7324c7246317029149c491
|
7
|
+
data.tar.gz: b278441842361cc53836fce087e87e6d2d800bccbe9b307c2f0b39d3c4c357aae96275d2bfb808ab2455ebce692948d1241f4f991a5036473da1d31da699832b
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: linux
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0' ]
|
12
|
+
os:
|
13
|
+
- ubuntu-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
data/ChangeLog
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
Release 0.16.0 - 2021/01/25
|
2
|
+
|
3
|
+
* input: Add `tag_source` and `record_tag_key` parameters for using record field as tag
|
4
|
+
* in_kafka_group: Use NumericParser for floating point
|
5
|
+
|
1
6
|
Release 0.15.3 - 2020/12/08
|
2
7
|
|
3
8
|
* in_kafka: Fix `record_time_key` parameter not working
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
14
|
gem.name = "fluent-plugin-kafka"
|
15
15
|
gem.require_paths = ["lib"]
|
16
|
-
gem.version = '0.
|
16
|
+
gem.version = '0.16.0'
|
17
17
|
gem.required_ruby_version = ">= 2.1.0"
|
18
18
|
|
19
19
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -21,4 +21,5 @@ Gem::Specification.new do |gem|
|
|
21
21
|
gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
|
22
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
23
23
|
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
24
|
+
gem.add_development_dependency "webrick"
|
24
25
|
end
|
@@ -31,6 +31,10 @@ class Fluent::KafkaInput < Fluent::Input
|
|
31
31
|
config_param :add_suffix, :string, :default => nil,
|
32
32
|
:desc => "tag suffix"
|
33
33
|
config_param :add_offset_in_record, :bool, :default => false
|
34
|
+
config_param :tag_source, :enum, :list => [:topic, :record], :default => :topic,
|
35
|
+
:desc => "Source for the fluentd event tag"
|
36
|
+
config_param :record_tag_key, :string, :default => 'tag',
|
37
|
+
:desc => "Tag field when tag_source is 'record'"
|
34
38
|
|
35
39
|
config_param :offset_zookeeper, :string, :default => nil
|
36
40
|
config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
|
@@ -225,6 +229,8 @@ class Fluent::KafkaInput < Fluent::Input
|
|
225
229
|
@kafka_message_key,
|
226
230
|
@time_source,
|
227
231
|
@record_time_key,
|
232
|
+
@tag_source,
|
233
|
+
@record_tag_key,
|
228
234
|
opt)
|
229
235
|
}
|
230
236
|
@topic_watchers.each {|tw|
|
@@ -249,7 +255,7 @@ class Fluent::KafkaInput < Fluent::Input
|
|
249
255
|
end
|
250
256
|
|
251
257
|
class TopicWatcher < Coolio::TimerWatcher
|
252
|
-
def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, record_time_key, options={})
|
258
|
+
def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, record_time_key, tag_source, record_tag_key, options={})
|
253
259
|
@topic_entry = topic_entry
|
254
260
|
@kafka = kafka
|
255
261
|
@callback = method(:consume)
|
@@ -262,6 +268,8 @@ class Fluent::KafkaInput < Fluent::Input
|
|
262
268
|
@kafka_message_key = kafka_message_key
|
263
269
|
@time_source = time_source
|
264
270
|
@record_time_key = record_time_key
|
271
|
+
@tag_source = tag_source
|
272
|
+
@record_tag_key = record_tag_key
|
265
273
|
|
266
274
|
@next_offset = @topic_entry.offset
|
267
275
|
if @topic_entry.offset == -1 && offset_manager
|
@@ -298,6 +306,11 @@ class Fluent::KafkaInput < Fluent::Input
|
|
298
306
|
messages.each { |msg|
|
299
307
|
begin
|
300
308
|
record = @parser.call(msg, @topic_entry)
|
309
|
+
if @tag_source == :record
|
310
|
+
tag = record[@record_tag_key]
|
311
|
+
tag = @add_prefix + "." + tag if @add_prefix
|
312
|
+
tag = tag + "." + @add_suffix if @add_suffix
|
313
|
+
end
|
301
314
|
case @time_source
|
302
315
|
when :kafka
|
303
316
|
record_time = Fluent::EventTime.from_time(msg.create_time)
|
@@ -36,6 +36,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
|
|
36
36
|
config_param :get_kafka_client_log, :bool, :default => false
|
37
37
|
config_param :time_format, :string, :default => nil,
|
38
38
|
:desc => "Time format to be used to parse 'time' field."
|
39
|
+
config_param :tag_source, :enum, :list => [:topic, :record], :default => :topic,
|
40
|
+
:desc => "Source for the fluentd event tag"
|
41
|
+
config_param :record_tag_key, :string, :default => 'tag',
|
42
|
+
:desc => "Tag field when tag_source is 'record'"
|
39
43
|
config_param :kafka_message_key, :string, :default => nil,
|
40
44
|
:desc => "Set kafka's message key to this field"
|
41
45
|
config_param :connect_timeout, :integer, :default => nil,
|
@@ -138,6 +142,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
|
|
138
142
|
@time_parser = Fluent::TextParser::TimeParser.new(@time_format)
|
139
143
|
end
|
140
144
|
end
|
145
|
+
|
146
|
+
if @time_source == :record && defined?(Fluent::NumericTimeParser)
|
147
|
+
@float_numeric_parse = Fluent::NumericTimeParser.new(:float)
|
148
|
+
end
|
141
149
|
end
|
142
150
|
|
143
151
|
def setup_parser(conf)
|
@@ -244,49 +252,104 @@ class Fluent::KafkaGroupInput < Fluent::Input
|
|
244
252
|
end
|
245
253
|
end
|
246
254
|
|
255
|
+
def process_batch_with_record_tag(batch)
|
256
|
+
es = {}
|
257
|
+
batch.messages.each { |msg|
|
258
|
+
begin
|
259
|
+
record = @parser_proc.call(msg)
|
260
|
+
tag = record[@record_tag_key]
|
261
|
+
tag = @add_prefix + "." + tag if @add_prefix
|
262
|
+
tag = tag + "." + @add_suffix if @add_suffix
|
263
|
+
es[tag] ||= Fluent::MultiEventStream.new
|
264
|
+
case @time_source
|
265
|
+
when :kafka
|
266
|
+
record_time = Fluent::EventTime.from_time(msg.create_time)
|
267
|
+
when :now
|
268
|
+
record_time = Fluent::Engine.now
|
269
|
+
when :record
|
270
|
+
if @time_format
|
271
|
+
record_time = @time_parser.parse(record[@record_time_key].to_s)
|
272
|
+
else
|
273
|
+
record_time = record[@record_time_key]
|
274
|
+
end
|
275
|
+
else
|
276
|
+
log.fatal "BUG: invalid time_source: #{@time_source}"
|
277
|
+
end
|
278
|
+
if @kafka_message_key
|
279
|
+
record[@kafka_message_key] = msg.key
|
280
|
+
end
|
281
|
+
if @add_headers
|
282
|
+
msg.headers.each_pair { |k, v|
|
283
|
+
record[k] = v
|
284
|
+
}
|
285
|
+
end
|
286
|
+
es[tag].add(record_time, record)
|
287
|
+
rescue => e
|
288
|
+
log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
|
289
|
+
log.debug_backtrace
|
290
|
+
end
|
291
|
+
}
|
292
|
+
|
293
|
+
unless es.empty?
|
294
|
+
es.each { |tag,es|
|
295
|
+
emit_events(tag, es)
|
296
|
+
}
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
def process_batch(batch)
|
301
|
+
es = Fluent::MultiEventStream.new
|
302
|
+
tag = batch.topic
|
303
|
+
tag = @add_prefix + "." + tag if @add_prefix
|
304
|
+
tag = tag + "." + @add_suffix if @add_suffix
|
305
|
+
|
306
|
+
batch.messages.each { |msg|
|
307
|
+
begin
|
308
|
+
record = @parser_proc.call(msg)
|
309
|
+
case @time_source
|
310
|
+
when :kafka
|
311
|
+
record_time = Fluent::EventTime.from_time(msg.create_time)
|
312
|
+
when :now
|
313
|
+
record_time = Fluent::Engine.now
|
314
|
+
when :record
|
315
|
+
record_time = record[@record_time_key]
|
316
|
+
|
317
|
+
if @time_format
|
318
|
+
record_time = @time_parser.parse(record_time.to_s)
|
319
|
+
elsif record_time.is_a?(Float) && @float_numeric_parse
|
320
|
+
record_time = @float_numeric_parse.parse(record_time)
|
321
|
+
end
|
322
|
+
else
|
323
|
+
log.fatal "BUG: invalid time_source: #{@time_source}"
|
324
|
+
end
|
325
|
+
if @kafka_message_key
|
326
|
+
record[@kafka_message_key] = msg.key
|
327
|
+
end
|
328
|
+
if @add_headers
|
329
|
+
msg.headers.each_pair { |k, v|
|
330
|
+
record[k] = v
|
331
|
+
}
|
332
|
+
end
|
333
|
+
es.add(record_time, record)
|
334
|
+
rescue => e
|
335
|
+
log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
|
336
|
+
log.debug_backtrace
|
337
|
+
end
|
338
|
+
}
|
339
|
+
|
340
|
+
unless es.empty?
|
341
|
+
emit_events(tag, es)
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
247
345
|
def run
|
248
346
|
while @consumer
|
249
347
|
begin
|
250
348
|
@consumer.each_batch(@fetch_opts) { |batch|
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
batch.messages.each { |msg|
|
257
|
-
begin
|
258
|
-
record = @parser_proc.call(msg)
|
259
|
-
case @time_source
|
260
|
-
when :kafka
|
261
|
-
record_time = Fluent::EventTime.from_time(msg.create_time)
|
262
|
-
when :now
|
263
|
-
record_time = Fluent::Engine.now
|
264
|
-
when :record
|
265
|
-
if @time_format
|
266
|
-
record_time = @time_parser.parse(record[@record_time_key].to_s)
|
267
|
-
else
|
268
|
-
record_time = record[@record_time_key]
|
269
|
-
end
|
270
|
-
else
|
271
|
-
log.fatal "BUG: invalid time_source: #{@time_source}"
|
272
|
-
end
|
273
|
-
if @kafka_message_key
|
274
|
-
record[@kafka_message_key] = msg.key
|
275
|
-
end
|
276
|
-
if @add_headers
|
277
|
-
msg.headers.each_pair { |k, v|
|
278
|
-
record[k] = v
|
279
|
-
}
|
280
|
-
end
|
281
|
-
es.add(record_time, record)
|
282
|
-
rescue => e
|
283
|
-
log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
|
284
|
-
log.debug_backtrace
|
285
|
-
end
|
286
|
-
}
|
287
|
-
|
288
|
-
unless es.empty?
|
289
|
-
emit_events(tag, es)
|
349
|
+
if @tag_source == :record
|
350
|
+
process_batch_with_record_tag(batch)
|
351
|
+
else
|
352
|
+
process_batch(batch)
|
290
353
|
end
|
291
354
|
}
|
292
355
|
rescue ForShutdown
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-01-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -93,6 +93,20 @@ dependencies:
|
|
93
93
|
- - ">="
|
94
94
|
- !ruby/object:Gem::Version
|
95
95
|
version: 3.0.8
|
96
|
+
- !ruby/object:Gem::Dependency
|
97
|
+
name: webrick
|
98
|
+
requirement: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
type: :development
|
104
|
+
prerelease: false
|
105
|
+
version_requirements: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
96
110
|
description: Fluentd plugin for Apache Kafka > 0.8
|
97
111
|
email:
|
98
112
|
- togachiro@gmail.com
|
@@ -101,6 +115,7 @@ executables: []
|
|
101
115
|
extensions: []
|
102
116
|
extra_rdoc_files: []
|
103
117
|
files:
|
118
|
+
- ".github/workflows/linux.yml"
|
104
119
|
- ".gitignore"
|
105
120
|
- ".travis.yml"
|
106
121
|
- ChangeLog
|