fluent-plugin-kafka 0.2.2 → 0.3.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/fluent-plugin-kafka.gemspec +2 -3
- data/lib/fluent/plugin/in_kafka.rb +99 -89
- data/lib/fluent/plugin/in_kafka_group.rb +68 -100
- data/lib/fluent/plugin/kafka_plugin_util.rb +21 -0
- data/lib/fluent/plugin/out_kafka.rb +72 -59
- metadata +7 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 64803e0c72fc4c81fb01fb053436a944c768ba6a
|
4
|
+
data.tar.gz: 602f630699406dfeef4712584cbdb95f1f06483f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9e80898dbac167495ceaf99ef48c8c077974190da51173e97a87c69e7064da9213781d2b9762a68f12f4fd87ba603a5b9fca92f562c2880a109e13104445c890
|
7
|
+
data.tar.gz: 88c81bc3ca6e27bead1010ff6112cf1d80fcd8de665e46c030eada336704d0ea33effc2bc101aa96622463b069bdc8aeffff2c3222395f6321b380bff0e9a8e3
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,12 +12,11 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.
|
15
|
+
gem.version = '0.3.0.rc1'
|
16
16
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
17
|
-
gem.add_dependency 'poseidon_cluster'
|
18
17
|
gem.add_dependency 'ltsv'
|
19
18
|
gem.add_dependency 'zookeeper'
|
20
|
-
gem.add_dependency 'ruby-kafka', '~> 0.3.
|
19
|
+
gem.add_dependency 'ruby-kafka', '~> 0.3.11'
|
21
20
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
22
21
|
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
23
22
|
end
|
@@ -1,4 +1,6 @@
|
|
1
1
|
require 'fluent/input'
|
2
|
+
require 'fluent/plugin/kafka_plugin_util'
|
3
|
+
|
2
4
|
module Fluent
|
3
5
|
|
4
6
|
class KafkaInput < Input
|
@@ -8,10 +10,12 @@ class KafkaInput < Input
|
|
8
10
|
:desc => "Supported format: (json|text|ltsv|msgpack)"
|
9
11
|
config_param :message_key, :string, :default => 'message',
|
10
12
|
:desc => "For 'text' format only."
|
11
|
-
config_param :host, :string, :default =>
|
13
|
+
config_param :host, :string, :default => nil,
|
12
14
|
:desc => "Broker host"
|
13
|
-
config_param :port, :integer, :default =>
|
15
|
+
config_param :port, :integer, :default => nil,
|
14
16
|
:desc => "Broker port"
|
17
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
18
|
+
:desc => "List of broker-host:port, separate with comma, must set."
|
15
19
|
config_param :interval, :integer, :default => 1, # seconds
|
16
20
|
:desc => "Interval (Unit: seconds)"
|
17
21
|
config_param :topics, :string, :default => nil,
|
@@ -30,24 +34,25 @@ class KafkaInput < Input
|
|
30
34
|
config_param :offset_zookeeper, :string, :default => nil
|
31
35
|
config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
|
32
36
|
|
33
|
-
#
|
37
|
+
# Kafka#fetch_messages options
|
34
38
|
config_param :max_bytes, :integer, :default => nil,
|
35
39
|
:desc => "Maximum number of bytes to fetch."
|
36
|
-
config_param :
|
40
|
+
config_param :max_wait_time, :integer, :default => nil,
|
37
41
|
:desc => "How long to block until the server sends us data."
|
38
42
|
config_param :min_bytes, :integer, :default => nil,
|
39
43
|
:desc => "Smallest amount of data the server should send us."
|
40
44
|
config_param :socket_timeout_ms, :integer, :default => nil,
|
41
45
|
:desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
|
42
46
|
|
47
|
+
include KafkaPluginUtil::SSLSettings
|
48
|
+
|
43
49
|
unless method_defined?(:router)
|
44
50
|
define_method("router") { Fluent::Engine }
|
45
51
|
end
|
46
52
|
|
47
53
|
def initialize
|
48
54
|
super
|
49
|
-
require '
|
50
|
-
require 'zookeeper'
|
55
|
+
require 'kafka'
|
51
56
|
end
|
52
57
|
|
53
58
|
def configure(conf)
|
@@ -73,38 +78,90 @@ class KafkaInput < Input
|
|
73
78
|
raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
|
74
79
|
end
|
75
80
|
|
81
|
+
# For backward compatibility
|
82
|
+
@brokers = case
|
83
|
+
when @host && @port
|
84
|
+
["#{@host}:#{@port}"]
|
85
|
+
when @host
|
86
|
+
["#{@host}:9092"]
|
87
|
+
when @port
|
88
|
+
["localhost:#{@port}"]
|
89
|
+
else
|
90
|
+
@brokers
|
91
|
+
end
|
92
|
+
|
93
|
+
if conf['max_wait_ms']
|
94
|
+
log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
|
95
|
+
@max_wait_time = conf['max_wait_ms'].to_i / 1000
|
96
|
+
end
|
97
|
+
|
98
|
+
@max_wait_time = @interval if @max_wait_time.nil?
|
99
|
+
|
100
|
+
require 'zookeeper' if @offset_zookeeper
|
101
|
+
|
102
|
+
@parser_proc = setup_parser
|
103
|
+
end
|
104
|
+
|
105
|
+
def setup_parser
|
76
106
|
case @format
|
77
107
|
when 'json'
|
78
108
|
require 'yajl'
|
109
|
+
Proc.new { |msg, te|
|
110
|
+
r = Yajl::Parser.parse(msg.value)
|
111
|
+
add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
|
112
|
+
r
|
113
|
+
}
|
79
114
|
when 'ltsv'
|
80
115
|
require 'ltsv'
|
116
|
+
Proc.new { |msg, te|
|
117
|
+
r = LTSV.parse(msg.value).first
|
118
|
+
add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
|
119
|
+
r
|
120
|
+
}
|
81
121
|
when 'msgpack'
|
82
122
|
require 'msgpack'
|
123
|
+
Proc.new { |msg, te|
|
124
|
+
r = MessagePack.unpack(msg.value)
|
125
|
+
add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
|
126
|
+
r
|
127
|
+
}
|
128
|
+
when 'text'
|
129
|
+
Proc.new { |msg, te|
|
130
|
+
r = {@message_key => msg.value}
|
131
|
+
add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
|
132
|
+
r
|
133
|
+
}
|
83
134
|
end
|
84
135
|
end
|
85
136
|
|
137
|
+
def add_offset_in_hash(hash, te, offset)
|
138
|
+
hash['kafka_topic'.freeze] = te.topic
|
139
|
+
hash['kafka_partition'.freeze] = te.partition
|
140
|
+
hash['kafka_offset'.freeze] = offset
|
141
|
+
end
|
142
|
+
|
86
143
|
def start
|
87
144
|
super
|
145
|
+
|
88
146
|
@loop = Coolio::Loop.new
|
89
147
|
opt = {}
|
90
148
|
opt[:max_bytes] = @max_bytes if @max_bytes
|
91
|
-
opt[:
|
149
|
+
opt[:max_wait_time] = @max_wait_time if @max_wait_time
|
92
150
|
opt[:min_bytes] = @min_bytes if @min_bytes
|
93
|
-
opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
|
94
151
|
|
152
|
+
@kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
|
153
|
+
ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
154
|
+
ssl_client_cert: read_ssl_file(@ssl_client_cert),
|
155
|
+
ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
|
95
156
|
@zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
|
96
157
|
|
97
158
|
@topic_watchers = @topic_list.map {|topic_entry|
|
98
159
|
offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper
|
99
160
|
TopicWatcher.new(
|
100
161
|
topic_entry,
|
101
|
-
@
|
102
|
-
@port,
|
103
|
-
@client_id,
|
162
|
+
@kafka,
|
104
163
|
interval,
|
105
|
-
@
|
106
|
-
@message_key,
|
107
|
-
@add_offset_in_record,
|
164
|
+
@parser_proc,
|
108
165
|
@add_prefix,
|
109
166
|
@add_suffix,
|
110
167
|
offset_manager,
|
@@ -120,26 +177,24 @@ class KafkaInput < Input
|
|
120
177
|
def shutdown
|
121
178
|
@loop.stop
|
122
179
|
@zookeeper.close! if @zookeeper
|
180
|
+
@thread.join
|
181
|
+
@kafka.close
|
123
182
|
super
|
124
183
|
end
|
125
184
|
|
126
185
|
def run
|
127
186
|
@loop.run
|
128
|
-
rescue
|
129
|
-
$log.error "unexpected error", :error
|
187
|
+
rescue => e
|
188
|
+
$log.error "unexpected error", :error => e.to_s
|
130
189
|
$log.error_backtrace
|
131
190
|
end
|
132
191
|
|
133
192
|
class TopicWatcher < Coolio::TimerWatcher
|
134
|
-
def initialize(topic_entry,
|
193
|
+
def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, options={})
|
135
194
|
@topic_entry = topic_entry
|
136
|
-
@
|
137
|
-
@port = port
|
138
|
-
@client_id = client_id
|
195
|
+
@kafka = kafka
|
139
196
|
@callback = method(:consume)
|
140
|
-
@
|
141
|
-
@message_key = message_key
|
142
|
-
@add_offset_in_record = add_offset_in_record
|
197
|
+
@parser = parser
|
143
198
|
@add_prefix = add_prefix
|
144
199
|
@add_suffix = add_suffix
|
145
200
|
@options = options
|
@@ -150,98 +205,53 @@ class KafkaInput < Input
|
|
150
205
|
if @topic_entry.offset == -1 && offset_manager
|
151
206
|
@next_offset = offset_manager.next_offset
|
152
207
|
end
|
153
|
-
@
|
208
|
+
@fetch_args = {
|
209
|
+
topic: @topic_entry.topic,
|
210
|
+
partition: @topic_entry.partition,
|
211
|
+
}.merge(@options)
|
154
212
|
|
155
213
|
super(interval, true)
|
156
214
|
end
|
157
215
|
|
158
216
|
def on_timer
|
159
217
|
@callback.call
|
160
|
-
rescue
|
161
|
-
|
162
|
-
|
163
|
-
|
218
|
+
rescue => e
|
219
|
+
# TODO log?
|
220
|
+
$log.error e.to_s
|
221
|
+
$log.error_backtrace
|
164
222
|
end
|
165
223
|
|
166
224
|
def consume
|
225
|
+
offset = @next_offset
|
226
|
+
@fetch_args[:offset] = offset
|
227
|
+
messages = @kafka.fetch_messages(@fetch_args)
|
228
|
+
|
229
|
+
return if messages.size.zero?
|
230
|
+
|
167
231
|
es = MultiEventStream.new
|
168
232
|
tag = @topic_entry.topic
|
169
233
|
tag = @add_prefix + "." + tag if @add_prefix
|
170
234
|
tag = tag + "." + @add_suffix if @add_suffix
|
171
235
|
|
172
|
-
|
173
|
-
@consumer = create_consumer(@next_offset)
|
174
|
-
end
|
175
|
-
|
176
|
-
@consumer.fetch.each { |msg|
|
236
|
+
messages.each { |msg|
|
177
237
|
begin
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
rescue
|
182
|
-
$log.warn msg_record.to_s, :error=>$!.to_s
|
238
|
+
es.add(Engine.now, @parser.call(msg, @topic_entry))
|
239
|
+
rescue => e
|
240
|
+
$log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
|
183
241
|
$log.debug_backtrace
|
184
242
|
end
|
185
243
|
}
|
244
|
+
offset = messages.last.offset + 1
|
186
245
|
|
187
246
|
unless es.empty?
|
188
247
|
@router.emit_stream(tag, es)
|
189
248
|
|
190
249
|
if @offset_manager
|
191
|
-
|
192
|
-
@offset_manager.save_offset(next_offset)
|
193
|
-
@next_offset = next_offset
|
250
|
+
@offset_manager.save_offset(offset)
|
194
251
|
end
|
252
|
+
@next_offset = offset
|
195
253
|
end
|
196
254
|
end
|
197
|
-
|
198
|
-
def create_consumer(offset)
|
199
|
-
@consumer.close if @consumer
|
200
|
-
Poseidon::PartitionConsumer.new(
|
201
|
-
@client_id, # client_id
|
202
|
-
@host, # host
|
203
|
-
@port, # port
|
204
|
-
@topic_entry.topic, # topic
|
205
|
-
@topic_entry.partition, # partition
|
206
|
-
offset, # offset
|
207
|
-
@options # options
|
208
|
-
)
|
209
|
-
end
|
210
|
-
|
211
|
-
def parse_line(record)
|
212
|
-
case @format
|
213
|
-
when 'json'
|
214
|
-
Yajl::Parser.parse(record)
|
215
|
-
when 'ltsv'
|
216
|
-
LTSV.parse(record)
|
217
|
-
when 'msgpack'
|
218
|
-
MessagePack.unpack(record)
|
219
|
-
when 'text'
|
220
|
-
{@message_key => record}
|
221
|
-
end
|
222
|
-
end
|
223
|
-
|
224
|
-
def decorate_offset(record, offset)
|
225
|
-
case @format
|
226
|
-
when 'json'
|
227
|
-
add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
|
228
|
-
when 'ltsv'
|
229
|
-
record.each { |line|
|
230
|
-
add_offset_in_hash(line, @topic_entry.topic, @topic_entry.partition, offset)
|
231
|
-
}
|
232
|
-
when 'msgpack'
|
233
|
-
add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
|
234
|
-
when 'text'
|
235
|
-
add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
|
236
|
-
end
|
237
|
-
record
|
238
|
-
end
|
239
|
-
|
240
|
-
def add_offset_in_hash(hash, topic, partition, offset)
|
241
|
-
hash['kafka_topic'] = topic
|
242
|
-
hash['kafka_partition'] = partition
|
243
|
-
hash['kafka_offset'] = offset
|
244
|
-
end
|
245
255
|
end
|
246
256
|
|
247
257
|
class TopicEntry
|
@@ -1,19 +1,17 @@
|
|
1
1
|
require 'fluent/input'
|
2
|
+
require 'fluent/plugin/kafka_plugin_util'
|
3
|
+
|
2
4
|
module Fluent
|
3
5
|
|
4
6
|
class KafkaGroupInput < Input
|
5
7
|
Plugin.register_input('kafka_group', self)
|
6
8
|
|
7
|
-
config_param :brokers, :string,
|
8
|
-
:desc => "List of broker-host:port, separate with comma, must set."
|
9
|
-
config_param :zookeepers, :string,
|
9
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
10
10
|
:desc => "List of broker-host:port, separate with comma, must set."
|
11
|
-
config_param :consumer_group, :string,
|
11
|
+
config_param :consumer_group, :string,
|
12
12
|
:desc => "Consumer group name, must set."
|
13
13
|
config_param :topics, :string,
|
14
14
|
:desc => "Listening topics(separate with comma',')."
|
15
|
-
config_param :interval, :integer, :default => 1, # seconds
|
16
|
-
:desc => "Interval (Unit: seconds)"
|
17
15
|
config_param :format, :string, :default => 'json',
|
18
16
|
:desc => "Supported format: (json|text|ltsv|msgpack)"
|
19
17
|
config_param :message_key, :string, :default => 'message',
|
@@ -23,15 +21,21 @@ class KafkaGroupInput < Input
|
|
23
21
|
config_param :add_suffix, :string, :default => nil,
|
24
22
|
:desc => "Tag suffix (Optional)"
|
25
23
|
|
26
|
-
#
|
27
|
-
config_param :
|
28
|
-
:desc => "Maximum number of bytes to fetch."
|
29
|
-
config_param :max_wait_ms, :integer, :default => nil,
|
24
|
+
# Kafka consumer options
|
25
|
+
config_param :max_wait_time, :integer, :default => nil,
|
30
26
|
:desc => "How long to block until the server sends us data."
|
31
27
|
config_param :min_bytes, :integer, :default => nil,
|
32
28
|
:desc => "Smallest amount of data the server should send us."
|
33
|
-
config_param :
|
34
|
-
:desc => "
|
29
|
+
config_param :session_timeout, :integer, :default => nil,
|
30
|
+
:desc => "The number of seconds after which, if a client hasn't contacted the Kafka cluster"
|
31
|
+
config_param :offset_commit_interval, :integer, :default => nil,
|
32
|
+
:desc => "The interval between offset commits, in seconds"
|
33
|
+
config_param :offset_commit_threshold, :integer, :default => nil,
|
34
|
+
:desc => "The number of messages that can be processed before their offsets are committed"
|
35
|
+
config_param :start_from_beginning, :bool, :default => true,
|
36
|
+
:desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
|
37
|
+
|
38
|
+
include KafkaPluginUtil::SSLSettings
|
35
39
|
|
36
40
|
unless method_defined?(:router)
|
37
41
|
define_method("router") { Fluent::Engine }
|
@@ -39,7 +43,7 @@ class KafkaGroupInput < Input
|
|
39
43
|
|
40
44
|
def initialize
|
41
45
|
super
|
42
|
-
require '
|
46
|
+
require 'kafka'
|
43
47
|
end
|
44
48
|
|
45
49
|
def _config_to_array(config)
|
@@ -54,125 +58,89 @@ class KafkaGroupInput < Input
|
|
54
58
|
|
55
59
|
def configure(conf)
|
56
60
|
super
|
57
|
-
@broker_list = _config_to_array(@brokers)
|
58
|
-
@zookeeper_list = _config_to_array(@zookeepers)
|
59
|
-
@topic_list = _config_to_array(@topics)
|
60
61
|
|
61
|
-
|
62
|
-
|
62
|
+
$log.info "Will watch for topics #{@topics} at brokers " \
|
63
|
+
"#{@brokers} and '#{@consumer_group}' group"
|
64
|
+
|
65
|
+
@topics = _config_to_array(@topics)
|
66
|
+
|
67
|
+
if conf['max_wait_ms']
|
68
|
+
log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
|
69
|
+
@max_wait_time = conf['max_wait_ms'].to_i / 1000
|
63
70
|
end
|
64
|
-
$log.info "Will watch for topics #{@topic_list} at brokers " \
|
65
|
-
"#{@broker_list}, zookeepers #{@zookeeper_list} and group " \
|
66
|
-
"'#{@consumer_group}'"
|
67
71
|
|
72
|
+
@parser_proc = setup_parser
|
73
|
+
end
|
74
|
+
|
75
|
+
def setup_parser
|
68
76
|
case @format
|
69
77
|
when 'json'
|
70
78
|
require 'yajl'
|
79
|
+
Proc.new { |msg| Yajl::Parser.parse(msg.value) }
|
71
80
|
when 'ltsv'
|
72
81
|
require 'ltsv'
|
82
|
+
Proc.new { |msg| LTSV.parse(msg.value).first }
|
73
83
|
when 'msgpack'
|
74
84
|
require 'msgpack'
|
85
|
+
Proc.new { |msg| MessagePack.unpack(msg.value) }
|
86
|
+
when 'text'
|
87
|
+
Proc.new { |msg| {@message_key => msg.value} }
|
75
88
|
end
|
76
89
|
end
|
77
90
|
|
78
91
|
def start
|
79
92
|
super
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
@
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
93
|
+
|
94
|
+
consumer_opts = {:group_id => @consumer_group}
|
95
|
+
consumer_opts[:session_timeout] = @session_timeout if @session_timeout
|
96
|
+
consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
|
97
|
+
consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
|
98
|
+
|
99
|
+
@fetch_opts = {}
|
100
|
+
@fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
|
101
|
+
@fetch_opts[:min_bytes] = @min_bytes if @min_bytes
|
102
|
+
|
103
|
+
@kafka = Kafka.new(seed_brokers: @brokers,
|
104
|
+
ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
105
|
+
ssl_client_cert: read_ssl_file(@ssl_client_cert),
|
106
|
+
ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
|
107
|
+
@consumer = @kafka.consumer(consumer_opts)
|
108
|
+
@topics.each { |topic|
|
109
|
+
@consumer.subscribe(topic, start_from_beginning: @start_from_beginning)
|
94
110
|
}
|
95
111
|
@thread = Thread.new(&method(:run))
|
96
112
|
end
|
97
113
|
|
98
114
|
def shutdown
|
99
|
-
@
|
115
|
+
@consumer.stop
|
116
|
+
@thread.join
|
117
|
+
@kafka.close
|
100
118
|
super
|
101
119
|
end
|
102
120
|
|
103
121
|
def run
|
104
|
-
@
|
105
|
-
rescue
|
106
|
-
$log.error "unexpected error", :error=>$!.to_s
|
107
|
-
$log.error_backtrace
|
108
|
-
end
|
109
|
-
|
110
|
-
class TopicWatcher < Coolio::TimerWatcher
|
111
|
-
def initialize(topic, broker_list, zookeeper_list, consumer_group,
|
112
|
-
interval, format, message_key, add_prefix, add_suffix,
|
113
|
-
router, options)
|
114
|
-
@topic = topic
|
115
|
-
@callback = method(:consume)
|
116
|
-
@format = format
|
117
|
-
@message_key = message_key
|
118
|
-
@add_prefix = add_prefix
|
119
|
-
@add_suffix = add_suffix
|
120
|
-
@router = router
|
121
|
-
|
122
|
-
@consumer = Poseidon::ConsumerGroup.new(
|
123
|
-
consumer_group,
|
124
|
-
broker_list,
|
125
|
-
zookeeper_list,
|
126
|
-
topic,
|
127
|
-
options
|
128
|
-
)
|
129
|
-
|
130
|
-
super(interval, true)
|
131
|
-
end
|
132
|
-
|
133
|
-
def on_timer
|
134
|
-
@callback.call
|
135
|
-
rescue
|
136
|
-
# TODO log?
|
137
|
-
$log.error $!.to_s
|
138
|
-
$log.error_backtrace
|
139
|
-
end
|
140
|
-
|
141
|
-
def consume
|
122
|
+
@consumer.each_batch(@fetch_opts) { |batch|
|
142
123
|
es = MultiEventStream.new
|
143
|
-
tag =
|
124
|
+
tag = batch.topic
|
144
125
|
tag = @add_prefix + "." + tag if @add_prefix
|
145
126
|
tag = tag + "." + @add_suffix if @add_suffix
|
146
127
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
$log.warn msg_record.to_s, :error=>$!.to_s
|
154
|
-
$log.debug_backtrace
|
155
|
-
end
|
128
|
+
batch.messages.each { |msg|
|
129
|
+
begin
|
130
|
+
es.add(Engine.now, @parser_proc.call(msg))
|
131
|
+
rescue => e
|
132
|
+
$log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
|
133
|
+
$log.debug_backtrace
|
156
134
|
end
|
157
|
-
|
135
|
+
}
|
158
136
|
|
159
137
|
unless es.empty?
|
160
|
-
|
138
|
+
router.emit_stream(tag, es)
|
161
139
|
end
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
when 'json'
|
167
|
-
Yajl::Parser.parse(record)
|
168
|
-
when 'ltsv'
|
169
|
-
LTSV.parse(record)
|
170
|
-
when 'msgpack'
|
171
|
-
MessagePack.unpack(record)
|
172
|
-
when 'text'
|
173
|
-
{@message_key => record}
|
174
|
-
end
|
175
|
-
end
|
140
|
+
}
|
141
|
+
rescue => e
|
142
|
+
$log.error "unexpected error", :error => e.to_s
|
143
|
+
$log.error_backtrace
|
176
144
|
end
|
177
145
|
end
|
178
146
|
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Fluent
|
2
|
+
module KafkaPluginUtil
|
3
|
+
module SSLSettings
|
4
|
+
def self.included(klass)
|
5
|
+
klass.instance_eval {
|
6
|
+
config_param :ssl_ca_cert, :string, :default => nil,
|
7
|
+
:desc => "a PEM encoded CA cert to use with and SSL connection."
|
8
|
+
config_param :ssl_client_cert, :string, :default => nil,
|
9
|
+
:desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
|
10
|
+
config_param :ssl_client_cert_key, :string, :default => nil,
|
11
|
+
:desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
|
12
|
+
}
|
13
|
+
end
|
14
|
+
|
15
|
+
def read_ssl_file(path)
|
16
|
+
return nil if path.nil?
|
17
|
+
File.read(path)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -3,7 +3,10 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
3
3
|
|
4
4
|
def initialize
|
5
5
|
super
|
6
|
-
|
6
|
+
|
7
|
+
require 'kafka'
|
8
|
+
|
9
|
+
@kafka = nil
|
7
10
|
end
|
8
11
|
|
9
12
|
config_param :brokers, :string, :default => 'localhost:9092',
|
@@ -25,14 +28,22 @@ DESC
|
|
25
28
|
config_param :output_include_tag, :bool, :default => false
|
26
29
|
config_param :output_include_time, :bool, :default => false
|
27
30
|
|
31
|
+
# https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
|
32
|
+
config_param :ssl_ca_cert, :string, :default => nil,
|
33
|
+
:desc => "a PEM encoded CA cert to use with and SSL connection."
|
34
|
+
config_param :ssl_client_cert, :string, :default => nil,
|
35
|
+
:desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
|
36
|
+
config_param :ssl_client_cert_key, :string, :default => nil,
|
37
|
+
:desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
|
38
|
+
|
28
39
|
# poseidon producer options
|
29
|
-
config_param :max_send_retries, :integer, :default =>
|
40
|
+
config_param :max_send_retries, :integer, :default => 1,
|
30
41
|
:desc => "Number of times to retry sending of messages to a leader."
|
31
42
|
config_param :required_acks, :integer, :default => 0,
|
32
43
|
:desc => "The number of acks required per request."
|
33
|
-
config_param :
|
44
|
+
config_param :ack_timeout, :integer, :default => nil,
|
34
45
|
:desc => "How long the producer waits for acks."
|
35
|
-
config_param :compression_codec, :string, :default =>
|
46
|
+
config_param :compression_codec, :string, :default => nil,
|
36
47
|
:desc => "The codec the producer uses to compress messages."
|
37
48
|
|
38
49
|
config_param :time_format, :string, :default => nil
|
@@ -40,13 +51,11 @@ DESC
|
|
40
51
|
attr_accessor :output_data_type
|
41
52
|
attr_accessor :field_separator
|
42
53
|
|
43
|
-
@seed_brokers = []
|
44
|
-
|
45
54
|
unless method_defined?(:log)
|
46
55
|
define_method("log") { $log }
|
47
56
|
end
|
48
57
|
|
49
|
-
def
|
58
|
+
def refresh_client
|
50
59
|
if @zookeeper
|
51
60
|
@seed_brokers = []
|
52
61
|
z = Zookeeper.new(@zookeeper)
|
@@ -59,8 +68,9 @@ DESC
|
|
59
68
|
end
|
60
69
|
begin
|
61
70
|
if @seed_brokers.length > 0
|
62
|
-
@
|
63
|
-
|
71
|
+
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
72
|
+
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
|
73
|
+
log.info "initialized kafka producer: #{@client_id}"
|
64
74
|
else
|
65
75
|
log.warn "No brokers found on Zookeeper"
|
66
76
|
end
|
@@ -69,25 +79,24 @@ DESC
|
|
69
79
|
end
|
70
80
|
end
|
71
81
|
|
82
|
+
def read_ssl_file(path)
|
83
|
+
return nil if path.nil?
|
84
|
+
File.read(path)
|
85
|
+
end
|
86
|
+
|
72
87
|
def configure(conf)
|
73
88
|
super
|
89
|
+
|
74
90
|
if @zookeeper
|
75
91
|
require 'zookeeper'
|
76
|
-
require 'yajl'
|
77
92
|
else
|
78
93
|
@seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
|
79
94
|
log.info "brokers has been set directly: #{@seed_brokers}"
|
80
95
|
end
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
when 'json'
|
86
|
-
require 'yajl'
|
87
|
-
when 'ltsv'
|
88
|
-
require 'ltsv'
|
89
|
-
when 'msgpack'
|
90
|
-
require 'msgpack'
|
96
|
+
|
97
|
+
if conf['ack_timeout_ms']
|
98
|
+
log.warn "'ack_timeout_ms' parameter is deprecated. Use second unit 'ack_timeout' instead"
|
99
|
+
@ack_timeout = conf['ack_timeout_ms'].to_i / 1000
|
91
100
|
end
|
92
101
|
|
93
102
|
@f_separator = case @field_separator
|
@@ -97,56 +106,57 @@ DESC
|
|
97
106
|
else "\t"
|
98
107
|
end
|
99
108
|
|
100
|
-
@
|
101
|
-
nil
|
102
|
-
elsif @output_data_type == 'ltsv'
|
103
|
-
nil
|
104
|
-
elsif @output_data_type == 'msgpack'
|
105
|
-
nil
|
106
|
-
elsif @output_data_type =~ /^attr:(.*)$/
|
107
|
-
$1.split(',').map(&:strip).reject(&:empty?)
|
108
|
-
else
|
109
|
-
@formatter = Fluent::Plugin.new_formatter(@output_data_type)
|
110
|
-
@formatter.configure(conf)
|
111
|
-
nil
|
112
|
-
end
|
109
|
+
@formatter_proc = setup_formatter(conf)
|
113
110
|
|
111
|
+
@producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
|
112
|
+
@producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
|
113
|
+
@producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
|
114
114
|
end
|
115
115
|
|
116
116
|
def start
|
117
117
|
super
|
118
|
-
|
118
|
+
refresh_client
|
119
119
|
end
|
120
120
|
|
121
121
|
def shutdown
|
122
122
|
super
|
123
|
+
@kafka = nil
|
123
124
|
end
|
124
125
|
|
125
|
-
def
|
126
|
-
if @
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
else
|
126
|
+
def setup_formatter(conf)
|
127
|
+
if @output_data_type == 'json'
|
128
|
+
require 'yajl'
|
129
|
+
Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
|
130
|
+
elsif @output_data_type == 'ltsv'
|
131
|
+
require 'ltsv'
|
132
|
+
Proc.new { |tag, time, record| LTSV.dump(record) }
|
133
|
+
elsif @output_data_type == 'msgpack'
|
134
|
+
require 'msgpack'
|
135
|
+
Proc.new { |tag, time, record| record.to_msgpack }
|
136
|
+
elsif @output_data_type =~ /^attr:(.*)$/
|
137
|
+
@custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
|
138
138
|
@custom_attributes.unshift('time') if @output_include_time
|
139
139
|
@custom_attributes.unshift('tag') if @output_include_tag
|
140
|
-
|
141
|
-
|
142
|
-
|
140
|
+
Proc.new { |tag, time, record|
|
141
|
+
@custom_attributes.map { |attr|
|
142
|
+
record[attr].nil? ? '' : record[attr].to_s
|
143
|
+
}.join(@f_separator)
|
144
|
+
}
|
145
|
+
else
|
146
|
+
@formatter = Fluent::Plugin.new_formatter(@output_data_type)
|
147
|
+
@formatter.configure(conf)
|
148
|
+
@formatter.method(:format)
|
143
149
|
end
|
144
150
|
end
|
145
151
|
|
146
152
|
def emit(tag, es, chain)
|
147
153
|
begin
|
148
154
|
chain.next
|
149
|
-
|
155
|
+
|
156
|
+
# out_kafka is mainly for testing so don't need the performance unlike out_kafka_buffered.
|
157
|
+
producer = @kafka.producer(@producer_opts)
|
158
|
+
|
159
|
+
es.each do |time, record|
|
150
160
|
if @output_include_time
|
151
161
|
if @time_format
|
152
162
|
record['time'] = Time.at(time).strftime(@time_format)
|
@@ -155,17 +165,20 @@ DESC
|
|
155
165
|
end
|
156
166
|
end
|
157
167
|
record['tag'] = tag if @output_include_tag
|
158
|
-
topic = record['topic'] ||
|
168
|
+
topic = record['topic'] || @default_topic || tag
|
159
169
|
partition_key = record['partition_key'] || @default_partition_key
|
160
|
-
value = @
|
161
|
-
|
162
|
-
|
163
|
-
|
170
|
+
value = @formatter_proc.call(tag, time, record)
|
171
|
+
|
172
|
+
log.on_trace { log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") }
|
173
|
+
producer.produce(value, topic: topic, partition_key: partition_key)
|
164
174
|
end
|
175
|
+
|
176
|
+
producer.deliver_messages
|
177
|
+
producer.shutdown
|
165
178
|
rescue Exception => e
|
166
|
-
log.warn
|
167
|
-
|
168
|
-
|
179
|
+
log.warn "Send exception occurred: #{e}"
|
180
|
+
producer.shutdown if producer
|
181
|
+
refresh_client
|
169
182
|
raise e
|
170
183
|
end
|
171
184
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -30,20 +30,6 @@ dependencies:
|
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: poseidon_cluster
|
35
|
-
requirement: !ruby/object:Gem::Requirement
|
36
|
-
requirements:
|
37
|
-
- - ">="
|
38
|
-
- !ruby/object:Gem::Version
|
39
|
-
version: '0'
|
40
|
-
type: :runtime
|
41
|
-
prerelease: false
|
42
|
-
version_requirements: !ruby/object:Gem::Requirement
|
43
|
-
requirements:
|
44
|
-
- - ">="
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
version: '0'
|
47
33
|
- !ruby/object:Gem::Dependency
|
48
34
|
name: ltsv
|
49
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -78,14 +64,14 @@ dependencies:
|
|
78
64
|
requirements:
|
79
65
|
- - "~>"
|
80
66
|
- !ruby/object:Gem::Version
|
81
|
-
version: 0.3.
|
67
|
+
version: 0.3.11
|
82
68
|
type: :runtime
|
83
69
|
prerelease: false
|
84
70
|
version_requirements: !ruby/object:Gem::Requirement
|
85
71
|
requirements:
|
86
72
|
- - "~>"
|
87
73
|
- !ruby/object:Gem::Version
|
88
|
-
version: 0.3.
|
74
|
+
version: 0.3.11
|
89
75
|
- !ruby/object:Gem::Dependency
|
90
76
|
name: rake
|
91
77
|
requirement: !ruby/object:Gem::Requirement
|
@@ -130,6 +116,7 @@ files:
|
|
130
116
|
- fluent-plugin-kafka.gemspec
|
131
117
|
- lib/fluent/plugin/in_kafka.rb
|
132
118
|
- lib/fluent/plugin/in_kafka_group.rb
|
119
|
+
- lib/fluent/plugin/kafka_plugin_util.rb
|
133
120
|
- lib/fluent/plugin/kafka_producer_ext.rb
|
134
121
|
- lib/fluent/plugin/out_kafka.rb
|
135
122
|
- lib/fluent/plugin/out_kafka_buffered.rb
|
@@ -149,9 +136,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
149
136
|
version: '0'
|
150
137
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
138
|
requirements:
|
152
|
-
- - "
|
139
|
+
- - ">"
|
153
140
|
- !ruby/object:Gem::Version
|
154
|
-
version:
|
141
|
+
version: 1.3.1
|
155
142
|
requirements: []
|
156
143
|
rubyforge_project:
|
157
144
|
rubygems_version: 2.5.1
|