fluent-plugin-kafka 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 724da4a2036ca9c067bb14f5dce7a4011120b486
4
- data.tar.gz: dcf10e1242df2a44b7cab4ac3ca48df545131bdb
3
+ metadata.gz: c15b97c9566c5fcae1478eaeef953786e4f58412
4
+ data.tar.gz: 0a12eca981a1a4e44227b33142dac987a8e36c96
5
5
  SHA512:
6
- metadata.gz: 5fa1fdc5af0678ea8212488021b261c5c1f06b129c6a88b796c24cc6dcc306647f770cae276003249e0f1b36a27a5f0d95e085cf2af2e6af0ebf7fad1a6a4257
7
- data.tar.gz: e304d72397ffb647e511d62e53ce467083f583f67e020ce376fc4d1270a957f47908b0204a27184a12b4c9f7b96a2ae724ca7e15872235663ef6c840d5de4e67
6
+ metadata.gz: c190e02a5a0aafd3784b0eab74e1adbc1a291347e99d5154446253f7116a22b4c1ba969bea27da76ed918a156417b3150eb1fc45b73c7cf2a3e777cc0ae0c4b2
7
+ data.tar.gz: 13f146fffce6d05a6519c0e7dd8e04c8dde01d85345f9dcb3c3ab8de4792d6666d8deb704422fc8cea93c95786308ef773decfb31d2704e8b61542ffa0ef2454
data/ChangeLog CHANGED
@@ -1,3 +1,10 @@
1
+ Release 0.6.1 - 2017/08/30
2
+
3
+ * Add stats and datadog monitoring support
4
+ * ssl_ca_certs now accepts multiple paths
5
+ * Fix bug by ruby-kafka 0.4.1 changes
6
+ * Update ruby-kafka dependency to v0.4.1
7
+
1
8
  Release 0.6.0 - 2017/07/25
2
9
 
3
10
  * Add principal and keytab parameters for SASL support
data/README.md CHANGED
@@ -153,7 +153,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
153
153
  kafka_agg_max_bytes (integer) :default => 4096
154
154
  kafka_agg_max_messages (integer) :default => nil (No limit)
155
155
  max_send_limit_bytes (integer) :default => nil (No drop)
156
- discard_kafka_delivery_failed (bool) :default => false (No discard)
156
+ discard_kafka_delivery_failed (bool) :default => false (No discard)
157
+ monitoring_list (array) :default => []
157
158
  </match>
158
159
 
159
160
  `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
@@ -172,6 +173,9 @@ Supports following ruby-kafka's producer options.
172
173
  - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
173
174
  - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
174
175
  - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
176
+ - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
177
+
178
+ If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
175
179
 
176
180
  See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
177
181
 
@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.6.0'
15
+ gem.version = '0.6.1'
16
16
  gem.required_ruby_version = ">= 2.1.0"
17
17
 
18
18
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
19
19
  gem.add_dependency 'ltsv'
20
- gem.add_dependency 'ruby-kafka', '~> 0.4.0'
20
+ gem.add_dependency 'ruby-kafka', '~> 0.4.1'
21
21
  gem.add_development_dependency "rake", ">= 0.9.2"
22
22
  gem.add_development_dependency "test-unit", ">= 3.0.8"
23
23
  end
@@ -4,7 +4,7 @@ module Fluent
4
4
  def self.included(klass)
5
5
  klass.instance_eval {
6
6
  # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
7
- config_param :ssl_ca_cert, :string, :default => nil,
7
+ config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
8
8
  :desc => "a PEM encoded CA cert to use with and SSL connection."
9
9
  config_param :ssl_client_cert, :string, :default => nil,
10
10
  :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
@@ -15,7 +15,12 @@ module Fluent
15
15
 
16
16
  def read_ssl_file(path)
17
17
  return nil if path.nil?
18
- File.read(path)
18
+
19
+ if path.is_a?(Array)
20
+ path.map { |fp| File.read(fp) }
21
+ else
22
+ File.read(path)
23
+ end
19
24
  end
20
25
  end
21
26
 
@@ -19,8 +19,7 @@ module Kafka
19
19
  topic,
20
20
  partition,
21
21
  partition_key,
22
- create_time,
23
- key.to_s.bytesize + value.to_s.bytesize
22
+ create_time
24
23
  )
25
24
 
26
25
  @target_topics.add(topic)
@@ -88,8 +87,7 @@ module Kafka
88
87
  @topic,
89
88
  partition,
90
89
  partition_key,
91
- create_time,
92
- key.to_s.bytesize + value.to_s.bytesize
90
+ create_time
93
91
  )
94
92
 
95
93
  @pending_message_queue.write(message)
@@ -101,6 +101,16 @@ DESC
101
101
  end
102
102
  @formatter_proc = setup_formatter(formatter_conf)
103
103
 
104
+ if @default_topic.nil?
105
+ if @chunk_keys.include?('topic') && !@chunk_keys.include?('tag')
106
+ log.warn "Use 'topic' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer topic,tag>"
107
+ end
108
+ else
109
+ if @chunk_keys.include?('tag')
110
+ log.warn "default_topic is set. Fluentd's event tag is not used for topic"
111
+ end
112
+ end
113
+
104
114
  @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
105
115
  @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
106
116
  @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
@@ -74,6 +74,9 @@ Add a regular expression to capture ActiveSupport notifications from the Kafka c
74
74
  requires activesupport gem - records will be generated under fluent_kafka_stats.**
75
75
  DESC
76
76
 
77
+ config_param :monitoring_list, :array, :default => [],
78
+ :desc => "library to be used to monitor. statsd and datadog are supported"
79
+
77
80
  include Fluent::KafkaPluginUtil::SSLSettings
78
81
  include Fluent::KafkaPluginUtil::SaslSettings
79
82
 
@@ -167,6 +170,11 @@ DESC
167
170
  @router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
168
171
  end
169
172
  end
173
+
174
+ @monitoring_list.each { |m|
175
+ require "kafka/#{m}"
176
+ log.info "#{m} monitoring started"
177
+ }
170
178
  end
171
179
 
172
180
  def start
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-07-25 00:00:00.000000000 Z
12
+ date: 2017-08-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -51,14 +51,14 @@ dependencies:
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: 0.4.0
54
+ version: 0.4.1
55
55
  type: :runtime
56
56
  prerelease: false
57
57
  version_requirements: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: 0.4.0
61
+ version: 0.4.1
62
62
  - !ruby/object:Gem::Dependency
63
63
  name: rake
64
64
  requirement: !ruby/object:Gem::Requirement