fluent-plugin-kafka 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 08281169934f59e651e0a601af17ab914d6a72af
4
- data.tar.gz: 2098afa88314d62e83729aad0b97be02461142a6
3
+ metadata.gz: dfe217eaf0f6bd1b83a27e0ab49bbc959b8aa311
4
+ data.tar.gz: d45ade2033759a9c41f2ec98354f792db1302e65
5
5
  SHA512:
6
- metadata.gz: b081eb740e9732e4dc919adc9166d3303df020af95bffbc01194e0eac1c477544a4c6c52b94affb71082ff1d8539e83d37e27cd4d0b696983065ba0f0a5433b4
7
- data.tar.gz: f95855d295f0527f68384d0b636ddae382b767a725fe521f4b6dd9cd3687a16d1ae3c3e0a90429c3b75cd2f088a1168c32e7989bb4b8cb35baee6a868444045f
6
+ metadata.gz: 04e02ee2614432dc8e0a07e99ef2cb25cf86e764b4e1c37e2b2b719e0e4592d890539b74d5b4ca2def0b3dace18770c28b7eb08bb797fead911b7e6188cf0083
7
+ data.tar.gz: 0192de37ab4a8408a8fe22effa2a71d7bf0864f553fcc9ea139d2a9146f5b8b26f3e7702d5579bff2b2e4f12b4a66f4b12bec66f7c0c9e45459d44c2e5571eaf
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.1.5'
15
+ gem.version = '0.2.0'
16
16
  gem.add_development_dependency 'test-unit'
17
17
  gem.add_dependency 'fluentd'
18
18
  gem.add_dependency 'poseidon_cluster'
@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
20
20
  gem.add_dependency 'yajl-ruby'
21
21
  gem.add_dependency 'msgpack'
22
22
  gem.add_dependency 'zookeeper'
23
+ gem.add_dependency 'ruby-kafka', '~> 0.3.2'
23
24
  end
@@ -1,10 +1,17 @@
1
1
  # encode: utf-8
2
+ require 'thread'
3
+
2
4
  class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
3
5
  Fluent::Plugin.register_output('kafka_buffered', self)
4
6
 
5
7
  def initialize
6
8
  super
7
- require 'poseidon'
9
+
10
+ require 'kafka'
11
+
12
+ @kafka = nil
13
+ @producers = {}
14
+ @producers_mutex = Mutex.new
8
15
  end
9
16
 
10
17
  config_param :brokers, :string, :default => 'localhost:9092',
@@ -32,17 +39,25 @@ DESC
32
39
  config_param :output_include_time, :bool, :default => false
33
40
  config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
34
41
 
42
+ # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
43
+ config_param :ssl_ca_cert, :string, :default => nil,
44
+ :desc => "a PEM encoded CA cert to use with and SSL connection."
45
+ config_param :ssl_client_cert, :string, :default => nil,
46
+ :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
47
+ config_param :ssl_client_cert_key, :string, :default => nil,
48
+ :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
49
+
35
50
  # poseidon producer options
36
- config_param :max_send_retries, :integer, :default => 3,
51
+ config_param :max_send_retries, :integer, :default => 1,
37
52
  :desc => "Number of times to retry sending of messages to a leader."
38
53
  config_param :required_acks, :integer, :default => 0,
39
54
  :desc => "The number of acks required per request."
40
- config_param :ack_timeout_ms, :integer, :default => 1500,
55
+ config_param :ack_timeout, :time, :default => nil,
41
56
  :desc => "How long the producer waits for acks."
42
- config_param :compression_codec, :string, :default => 'none',
57
+ config_param :compression_codec, :string, :default => nil,
43
58
  :desc => <<-DESC
44
59
  The codec the producer uses to compress messages.
45
- Supported codecs: (none|gzip|snappy)
60
+ Supported codecs: (gzip|snappy)
46
61
  DESC
47
62
 
48
63
  config_param :time_format, :string, :default => nil
@@ -54,9 +69,7 @@ DESC
54
69
  define_method("log") { $log }
55
70
  end
56
71
 
57
- @seed_brokers = []
58
-
59
- def refresh_producer()
72
+ def refresh_client(raise_error = true)
60
73
  if @zookeeper
61
74
  @seed_brokers = []
62
75
  z = Zookeeper.new(@zookeeper)
@@ -69,27 +82,39 @@ DESC
69
82
  end
70
83
  begin
71
84
  if @seed_brokers.length > 0
72
- @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
73
- log.info "initialized producer #{@client_id}"
85
+ @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
86
+ ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
87
+ log.info "initialized kafka producer: #{@client_id}"
74
88
  else
75
89
  log.warn "No brokers found on Zookeeper"
76
90
  end
77
91
  rescue Exception => e
78
- log.error e
92
+ if raise_error # During startup, error should be reported to engine and stop its phase for safety.
93
+ raise e
94
+ else
95
+ log.error e
96
+ end
79
97
  end
80
98
  end
81
99
 
100
+ def read_ssl_file(path)
101
+ return nil if path.nil?
102
+ File.read(path)
103
+ end
104
+
82
105
  def configure(conf)
83
106
  super
107
+
84
108
  if @zookeeper
85
109
  require 'zookeeper'
86
- require 'yajl'
87
110
  else
88
111
  @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
89
112
  log.info "brokers has been set directly: #{@seed_brokers}"
90
113
  end
91
- if @compression_codec == 'snappy'
92
- require 'snappy'
114
+
115
+ if conf['ack_timeout_ms']
116
+ log.warn "'ack_timeout_ms' parameter is deprecated. Use second unit 'ack_timeout' instead"
117
+ @ack_timeout = conf['ack_timeout_ms'].to_i / 1000
93
118
  end
94
119
 
95
120
  @f_separator = case @field_separator
@@ -100,21 +125,48 @@ DESC
100
125
  end
101
126
 
102
127
  @formatter_proc = setup_formatter(conf)
128
+
129
+ @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks,
130
+ max_buffer_size: @buffer.buffer_chunk_limit / 10, max_buffer_bytesize: @buffer.buffer_chunk_limit * 2}
131
+ @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
132
+ @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
103
133
  end
104
134
 
105
135
  def start
106
136
  super
107
- refresh_producer()
137
+ refresh_client
108
138
  end
109
139
 
110
140
  def shutdown
111
141
  super
142
+ shutdown_producers
143
+ @kafka = nil
112
144
  end
113
145
 
114
146
  def format(tag, time, record)
115
147
  [tag, time, record].to_msgpack
116
148
  end
117
149
 
150
+ def shutdown_producers
151
+ @producers_mutex.synchronize {
152
+ @producers.each { |key, producer|
153
+ producer.shutdown
154
+ }
155
+ @producers = {}
156
+ }
157
+ end
158
+
159
+ def get_producer
160
+ @producers_mutex.synchronize {
161
+ producer = @producers[Thread.current.object_id]
162
+ unless producer
163
+ producer = @kafka.producer(@producer_opts)
164
+ @producers[Thread.current.object_id] = producer
165
+ end
166
+ producer
167
+ }
168
+ end
169
+
118
170
  def setup_formatter(conf)
119
171
  if @output_data_type == 'json'
120
172
  require 'yajl'
@@ -144,9 +196,11 @@ DESC
144
196
  end
145
197
 
146
198
  def write(chunk)
199
+ producer = get_producer
200
+
147
201
  records_by_topic = {}
148
202
  bytes_by_topic = {}
149
- messages = []
203
+ messages = 0
150
204
  messages_bytes = 0
151
205
  begin
152
206
  chunk.msgpack_each { |tag, time, record|
@@ -167,29 +221,31 @@ DESC
167
221
 
168
222
  record_buf = @formatter_proc.call(tag, time, record)
169
223
  record_buf_bytes = record_buf.bytesize
170
- if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
171
- log.on_trace { log.trace("#{messages.length} messages send.") }
172
- @producer.send_messages(messages)
173
- messages = []
224
+ if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
225
+ log.on_trace { log.trace("#{messages} messages send.") }
226
+ producer.deliver_messages
227
+ messages = 0
174
228
  messages_bytes = 0
175
229
  end
176
230
  log.on_trace { log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.") }
177
- messages << Poseidon::MessageToSend.new(topic, record_buf, partition_key)
231
+ messages += 1
232
+ producer.produce(record_buf, topic: topic, partition_key: partition_key)
178
233
  messages_bytes += record_buf_bytes
179
234
 
180
235
  records_by_topic[topic] += 1
181
236
  bytes_by_topic[topic] += record_buf_bytes
182
237
  }
183
- if messages.length > 0
184
- log.trace("#{messages.length} messages send.")
185
- @producer.send_messages(messages)
238
+ if messages > 0
239
+ log.trace("#{messages} messages send.")
240
+ producer.deliver_messages
186
241
  end
187
242
  log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
188
243
  end
189
244
  rescue Exception => e
190
245
  log.warn "Send exception occurred: #{e}"
191
- @producer.close if @producer
192
- refresh_producer()
246
+ # For safety, refresh client and its producers
247
+ shutdown_producers
248
+ refresh_client(false)
193
249
  # Raise exception to retry sendind messages
194
250
  raise e
195
251
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-02 00:00:00.000000000 Z
11
+ date: 2016-06-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: ruby-kafka
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: 0.3.2
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: 0.3.2
111
125
  description: Fluentd plugin for Apache Kafka > 0.8
112
126
  email:
113
127
  - togachiro@gmail.com