fluent-plugin-kafka 0.6.0 → 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 724da4a2036ca9c067bb14f5dce7a4011120b486
4
- data.tar.gz: dcf10e1242df2a44b7cab4ac3ca48df545131bdb
3
+ metadata.gz: c15b97c9566c5fcae1478eaeef953786e4f58412
4
+ data.tar.gz: 0a12eca981a1a4e44227b33142dac987a8e36c96
5
5
  SHA512:
6
- metadata.gz: 5fa1fdc5af0678ea8212488021b261c5c1f06b129c6a88b796c24cc6dcc306647f770cae276003249e0f1b36a27a5f0d95e085cf2af2e6af0ebf7fad1a6a4257
7
- data.tar.gz: e304d72397ffb647e511d62e53ce467083f583f67e020ce376fc4d1270a957f47908b0204a27184a12b4c9f7b96a2ae724ca7e15872235663ef6c840d5de4e67
6
+ metadata.gz: c190e02a5a0aafd3784b0eab74e1adbc1a291347e99d5154446253f7116a22b4c1ba969bea27da76ed918a156417b3150eb1fc45b73c7cf2a3e777cc0ae0c4b2
7
+ data.tar.gz: 13f146fffce6d05a6519c0e7dd8e04c8dde01d85345f9dcb3c3ab8de4792d6666d8deb704422fc8cea93c95786308ef773decfb31d2704e8b61542ffa0ef2454
data/ChangeLog CHANGED
@@ -1,3 +1,10 @@
1
+ Release 0.6.1 - 2017/08/30
2
+
3
+ * Add stats and datadog monitoring support
4
+ * ssl_ca_certs now accepts multiple paths
5
+ * Fix bug by ruby-kafka 0.4.1 changes
6
+ * Update ruby-kafka dependency to v0.4.1
7
+
1
8
  Release 0.6.0 - 2017/07/25
2
9
 
3
10
  * Add principal and keytab parameters for SASL support
data/README.md CHANGED
@@ -153,7 +153,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
153
153
  kafka_agg_max_bytes (integer) :default => 4096
154
154
  kafka_agg_max_messages (integer) :default => nil (No limit)
155
155
  max_send_limit_bytes (integer) :default => nil (No drop)
156
- discard_kafka_delivery_failed (bool) :default => false (No discard)
156
+ discard_kafka_delivery_failed (bool) :default => false (No discard)
157
+ monitoring_list (array) :default => []
157
158
  </match>
158
159
 
159
160
  `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
@@ -172,6 +173,9 @@ Supports following ruby-kafka's producer options.
172
173
  - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
173
174
  - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
174
175
  - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
176
+ - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
177
+
178
+ If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
175
179
 
176
180
  See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
177
181
 
@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.6.0'
15
+ gem.version = '0.6.1'
16
16
  gem.required_ruby_version = ">= 2.1.0"
17
17
 
18
18
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
19
19
  gem.add_dependency 'ltsv'
20
- gem.add_dependency 'ruby-kafka', '~> 0.4.0'
20
+ gem.add_dependency 'ruby-kafka', '~> 0.4.1'
21
21
  gem.add_development_dependency "rake", ">= 0.9.2"
22
22
  gem.add_development_dependency "test-unit", ">= 3.0.8"
23
23
  end
@@ -4,7 +4,7 @@ module Fluent
4
4
  def self.included(klass)
5
5
  klass.instance_eval {
6
6
  # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
7
- config_param :ssl_ca_cert, :string, :default => nil,
7
+ config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
8
8
  :desc => "a PEM encoded CA cert to use with and SSL connection."
9
9
  config_param :ssl_client_cert, :string, :default => nil,
10
10
  :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
@@ -15,7 +15,12 @@ module Fluent
15
15
 
16
16
  def read_ssl_file(path)
17
17
  return nil if path.nil?
18
- File.read(path)
18
+
19
+ if path.is_a?(Array)
20
+ path.map { |fp| File.read(fp) }
21
+ else
22
+ File.read(path)
23
+ end
19
24
  end
20
25
  end
21
26
 
@@ -19,8 +19,7 @@ module Kafka
19
19
  topic,
20
20
  partition,
21
21
  partition_key,
22
- create_time,
23
- key.to_s.bytesize + value.to_s.bytesize
22
+ create_time
24
23
  )
25
24
 
26
25
  @target_topics.add(topic)
@@ -88,8 +87,7 @@ module Kafka
88
87
  @topic,
89
88
  partition,
90
89
  partition_key,
91
- create_time,
92
- key.to_s.bytesize + value.to_s.bytesize
90
+ create_time
93
91
  )
94
92
 
95
93
  @pending_message_queue.write(message)
@@ -101,6 +101,16 @@ DESC
101
101
  end
102
102
  @formatter_proc = setup_formatter(formatter_conf)
103
103
 
104
+ if @default_topic.nil?
105
+ if @chunk_keys.include?('topic') && !@chunk_keys.include?('tag')
106
+ log.warn "Use 'topic' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer topic,tag>"
107
+ end
108
+ else
109
+ if @chunk_keys.include?('tag')
110
+ log.warn "default_topic is set. Fluentd's event tag is not used for topic"
111
+ end
112
+ end
113
+
104
114
  @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
105
115
  @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
106
116
  @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
@@ -74,6 +74,9 @@ Add a regular expression to capture ActiveSupport notifications from the Kafka c
74
74
  requires activesupport gem - records will be generated under fluent_kafka_stats.**
75
75
  DESC
76
76
 
77
+ config_param :monitoring_list, :array, :default => [],
78
+ :desc => "library to be used to monitor. statsd and datadog are supported"
79
+
77
80
  include Fluent::KafkaPluginUtil::SSLSettings
78
81
  include Fluent::KafkaPluginUtil::SaslSettings
79
82
 
@@ -167,6 +170,11 @@ DESC
167
170
  @router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
168
171
  end
169
172
  end
173
+
174
+ @monitoring_list.each { |m|
175
+ require "kafka/#{m}"
176
+ log.info "#{m} monitoring started"
177
+ }
170
178
  end
171
179
 
172
180
  def start
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-07-25 00:00:00.000000000 Z
12
+ date: 2017-08-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -51,14 +51,14 @@ dependencies:
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: 0.4.0
54
+ version: 0.4.1
55
55
  type: :runtime
56
56
  prerelease: false
57
57
  version_requirements: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: 0.4.0
61
+ version: 0.4.1
62
62
  - !ruby/object:Gem::Dependency
63
63
  name: rake
64
64
  requirement: !ruby/object:Gem::Requirement