fluent-plugin-kafka 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +7 -0
- data/README.md +5 -1
- data/fluent-plugin-kafka.gemspec +2 -2
- data/lib/fluent/plugin/kafka_plugin_util.rb +7 -2
- data/lib/fluent/plugin/kafka_producer_ext.rb +2 -4
- data/lib/fluent/plugin/out_kafka2.rb +10 -0
- data/lib/fluent/plugin/out_kafka_buffered.rb +8 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c15b97c9566c5fcae1478eaeef953786e4f58412
|
4
|
+
data.tar.gz: 0a12eca981a1a4e44227b33142dac987a8e36c96
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c190e02a5a0aafd3784b0eab74e1adbc1a291347e99d5154446253f7116a22b4c1ba969bea27da76ed918a156417b3150eb1fc45b73c7cf2a3e777cc0ae0c4b2
|
7
|
+
data.tar.gz: 13f146fffce6d05a6519c0e7dd8e04c8dde01d85345f9dcb3c3ab8de4792d6666d8deb704422fc8cea93c95786308ef773decfb31d2704e8b61542ffa0ef2454
|
data/ChangeLog
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
Release 0.6.1 - 2017/08/30
|
2
|
+
|
3
|
+
* Add stats and datadog monitoring support
|
4
|
+
* ssl_ca_certs now accepts multiple paths
|
5
|
+
* Fix bug by ruby-kafka 0.4.1 changes
|
6
|
+
* Update ruby-kafka dependency to v0.4.1
|
7
|
+
|
1
8
|
Release 0.6.0 - 2017/07/25
|
2
9
|
|
3
10
|
* Add principal and keytab parameters for SASL support
|
data/README.md
CHANGED
@@ -153,7 +153,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
|
|
153
153
|
kafka_agg_max_bytes (integer) :default => 4096
|
154
154
|
kafka_agg_max_messages (integer) :default => nil (No limit)
|
155
155
|
max_send_limit_bytes (integer) :default => nil (No drop)
|
156
|
-
discard_kafka_delivery_failed (bool)
|
156
|
+
discard_kafka_delivery_failed (bool) :default => false (No discard)
|
157
|
+
monitoring_list (array) :default => []
|
157
158
|
</match>
|
158
159
|
|
159
160
|
`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
|
@@ -172,6 +173,9 @@ Supports following ruby-kafka's producer options.
|
|
172
173
|
- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
|
173
174
|
- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
|
174
175
|
- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
|
176
|
+
- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
|
177
|
+
|
178
|
+
If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
|
175
179
|
|
176
180
|
See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
|
177
181
|
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.6.
|
15
|
+
gem.version = '0.6.1'
|
16
16
|
gem.required_ruby_version = ">= 2.1.0"
|
17
17
|
|
18
18
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
19
19
|
gem.add_dependency 'ltsv'
|
20
|
-
gem.add_dependency 'ruby-kafka', '~> 0.4.
|
20
|
+
gem.add_dependency 'ruby-kafka', '~> 0.4.1'
|
21
21
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
22
22
|
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
23
23
|
end
|
@@ -4,7 +4,7 @@ module Fluent
|
|
4
4
|
def self.included(klass)
|
5
5
|
klass.instance_eval {
|
6
6
|
# https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
|
7
|
-
config_param :ssl_ca_cert, :string, :default => nil,
|
7
|
+
config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
|
8
8
|
:desc => "a PEM encoded CA cert to use with and SSL connection."
|
9
9
|
config_param :ssl_client_cert, :string, :default => nil,
|
10
10
|
:desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
|
@@ -15,7 +15,12 @@ module Fluent
|
|
15
15
|
|
16
16
|
def read_ssl_file(path)
|
17
17
|
return nil if path.nil?
|
18
|
-
|
18
|
+
|
19
|
+
if path.is_a?(Array)
|
20
|
+
path.map { |fp| File.read(fp) }
|
21
|
+
else
|
22
|
+
File.read(path)
|
23
|
+
end
|
19
24
|
end
|
20
25
|
end
|
21
26
|
|
@@ -19,8 +19,7 @@ module Kafka
|
|
19
19
|
topic,
|
20
20
|
partition,
|
21
21
|
partition_key,
|
22
|
-
create_time
|
23
|
-
key.to_s.bytesize + value.to_s.bytesize
|
22
|
+
create_time
|
24
23
|
)
|
25
24
|
|
26
25
|
@target_topics.add(topic)
|
@@ -88,8 +87,7 @@ module Kafka
|
|
88
87
|
@topic,
|
89
88
|
partition,
|
90
89
|
partition_key,
|
91
|
-
create_time
|
92
|
-
key.to_s.bytesize + value.to_s.bytesize
|
90
|
+
create_time
|
93
91
|
)
|
94
92
|
|
95
93
|
@pending_message_queue.write(message)
|
@@ -101,6 +101,16 @@ DESC
|
|
101
101
|
end
|
102
102
|
@formatter_proc = setup_formatter(formatter_conf)
|
103
103
|
|
104
|
+
if @default_topic.nil?
|
105
|
+
if @chunk_keys.include?('topic') && !@chunk_keys.include?('tag')
|
106
|
+
log.warn "Use 'topic' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer topic,tag>"
|
107
|
+
end
|
108
|
+
else
|
109
|
+
if @chunk_keys.include?('tag')
|
110
|
+
log.warn "default_topic is set. Fluentd's event tag is not used for topic"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
104
114
|
@producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
|
105
115
|
@producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
|
106
116
|
@producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
|
@@ -74,6 +74,9 @@ Add a regular expression to capture ActiveSupport notifications from the Kafka c
|
|
74
74
|
requires activesupport gem - records will be generated under fluent_kafka_stats.**
|
75
75
|
DESC
|
76
76
|
|
77
|
+
config_param :monitoring_list, :array, :default => [],
|
78
|
+
:desc => "library to be used to monitor. statsd and datadog are supported"
|
79
|
+
|
77
80
|
include Fluent::KafkaPluginUtil::SSLSettings
|
78
81
|
include Fluent::KafkaPluginUtil::SaslSettings
|
79
82
|
|
@@ -167,6 +170,11 @@ DESC
|
|
167
170
|
@router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
|
168
171
|
end
|
169
172
|
end
|
173
|
+
|
174
|
+
@monitoring_list.each { |m|
|
175
|
+
require "kafka/#{m}"
|
176
|
+
log.info "#{m} monitoring started"
|
177
|
+
}
|
170
178
|
end
|
171
179
|
|
172
180
|
def start
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-08-31 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -51,14 +51,14 @@ dependencies:
|
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.4.
|
54
|
+
version: 0.4.1
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 0.4.
|
61
|
+
version: 0.4.1
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: rake
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|