fluent-plugin-kafka 0.6.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +7 -0
- data/README.md +5 -1
- data/fluent-plugin-kafka.gemspec +2 -2
- data/lib/fluent/plugin/kafka_plugin_util.rb +7 -2
- data/lib/fluent/plugin/kafka_producer_ext.rb +2 -4
- data/lib/fluent/plugin/out_kafka2.rb +10 -0
- data/lib/fluent/plugin/out_kafka_buffered.rb +8 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c15b97c9566c5fcae1478eaeef953786e4f58412
|
4
|
+
data.tar.gz: 0a12eca981a1a4e44227b33142dac987a8e36c96
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c190e02a5a0aafd3784b0eab74e1adbc1a291347e99d5154446253f7116a22b4c1ba969bea27da76ed918a156417b3150eb1fc45b73c7cf2a3e777cc0ae0c4b2
|
7
|
+
data.tar.gz: 13f146fffce6d05a6519c0e7dd8e04c8dde01d85345f9dcb3c3ab8de4792d6666d8deb704422fc8cea93c95786308ef773decfb31d2704e8b61542ffa0ef2454
|
data/ChangeLog
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
Release 0.6.1 - 2017/08/30
|
2
|
+
|
3
|
+
* Add stats and datadog monitoring support
|
4
|
+
* ssl_ca_certs now accepts multiple paths
|
5
|
+
* Fix bug by ruby-kafka 0.4.1 changes
|
6
|
+
* Update ruby-kafka dependency to v0.4.1
|
7
|
+
|
1
8
|
Release 0.6.0 - 2017/07/25
|
2
9
|
|
3
10
|
* Add principal and keytab parameters for SASL support
|
data/README.md
CHANGED
@@ -153,7 +153,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
|
|
153
153
|
kafka_agg_max_bytes (integer) :default => 4096
|
154
154
|
kafka_agg_max_messages (integer) :default => nil (No limit)
|
155
155
|
max_send_limit_bytes (integer) :default => nil (No drop)
|
156
|
-
discard_kafka_delivery_failed (bool)
|
156
|
+
discard_kafka_delivery_failed (bool) :default => false (No discard)
|
157
|
+
monitoring_list (array) :default => []
|
157
158
|
</match>
|
158
159
|
|
159
160
|
`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
|
@@ -172,6 +173,9 @@ Supports following ruby-kafka's producer options.
|
|
172
173
|
- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
|
173
174
|
- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
|
174
175
|
- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
|
176
|
+
- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
|
177
|
+
|
178
|
+
If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
|
175
179
|
|
176
180
|
See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
|
177
181
|
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.6.
|
15
|
+
gem.version = '0.6.1'
|
16
16
|
gem.required_ruby_version = ">= 2.1.0"
|
17
17
|
|
18
18
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
19
19
|
gem.add_dependency 'ltsv'
|
20
|
-
gem.add_dependency 'ruby-kafka', '~> 0.4.
|
20
|
+
gem.add_dependency 'ruby-kafka', '~> 0.4.1'
|
21
21
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
22
22
|
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
23
23
|
end
|
@@ -4,7 +4,7 @@ module Fluent
|
|
4
4
|
def self.included(klass)
|
5
5
|
klass.instance_eval {
|
6
6
|
# https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
|
7
|
-
config_param :ssl_ca_cert, :string, :default => nil,
|
7
|
+
config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
|
8
8
|
:desc => "a PEM encoded CA cert to use with and SSL connection."
|
9
9
|
config_param :ssl_client_cert, :string, :default => nil,
|
10
10
|
:desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
|
@@ -15,7 +15,12 @@ module Fluent
|
|
15
15
|
|
16
16
|
def read_ssl_file(path)
|
17
17
|
return nil if path.nil?
|
18
|
-
|
18
|
+
|
19
|
+
if path.is_a?(Array)
|
20
|
+
path.map { |fp| File.read(fp) }
|
21
|
+
else
|
22
|
+
File.read(path)
|
23
|
+
end
|
19
24
|
end
|
20
25
|
end
|
21
26
|
|
@@ -19,8 +19,7 @@ module Kafka
|
|
19
19
|
topic,
|
20
20
|
partition,
|
21
21
|
partition_key,
|
22
|
-
create_time
|
23
|
-
key.to_s.bytesize + value.to_s.bytesize
|
22
|
+
create_time
|
24
23
|
)
|
25
24
|
|
26
25
|
@target_topics.add(topic)
|
@@ -88,8 +87,7 @@ module Kafka
|
|
88
87
|
@topic,
|
89
88
|
partition,
|
90
89
|
partition_key,
|
91
|
-
create_time
|
92
|
-
key.to_s.bytesize + value.to_s.bytesize
|
90
|
+
create_time
|
93
91
|
)
|
94
92
|
|
95
93
|
@pending_message_queue.write(message)
|
@@ -101,6 +101,16 @@ DESC
|
|
101
101
|
end
|
102
102
|
@formatter_proc = setup_formatter(formatter_conf)
|
103
103
|
|
104
|
+
if @default_topic.nil?
|
105
|
+
if @chunk_keys.include?('topic') && !@chunk_keys.include?('tag')
|
106
|
+
log.warn "Use 'topic' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer topic,tag>"
|
107
|
+
end
|
108
|
+
else
|
109
|
+
if @chunk_keys.include?('tag')
|
110
|
+
log.warn "default_topic is set. Fluentd's event tag is not used for topic"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
104
114
|
@producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
|
105
115
|
@producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
|
106
116
|
@producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
|
@@ -74,6 +74,9 @@ Add a regular expression to capture ActiveSupport notifications from the Kafka c
|
|
74
74
|
requires activesupport gem - records will be generated under fluent_kafka_stats.**
|
75
75
|
DESC
|
76
76
|
|
77
|
+
config_param :monitoring_list, :array, :default => [],
|
78
|
+
:desc => "library to be used to monitor. statsd and datadog are supported"
|
79
|
+
|
77
80
|
include Fluent::KafkaPluginUtil::SSLSettings
|
78
81
|
include Fluent::KafkaPluginUtil::SaslSettings
|
79
82
|
|
@@ -167,6 +170,11 @@ DESC
|
|
167
170
|
@router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
|
168
171
|
end
|
169
172
|
end
|
173
|
+
|
174
|
+
@monitoring_list.each { |m|
|
175
|
+
require "kafka/#{m}"
|
176
|
+
log.info "#{m} monitoring started"
|
177
|
+
}
|
170
178
|
end
|
171
179
|
|
172
180
|
def start
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-08-31 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -51,14 +51,14 @@ dependencies:
|
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.4.
|
54
|
+
version: 0.4.1
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 0.4.
|
61
|
+
version: 0.4.1
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: rake
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|