fluent-plugin-kafka 0.12.1 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 497c5a450bc4f55ddaf1b46454ed80f028c283775666e2a38ee0d91f16d391ef
-  data.tar.gz: 5f40c26e75f06e4ee73cc0b3ba32ef8438da48bea465549fb95f4bcb89e98819
+  metadata.gz: 05cbb3ff005fbf6f27ab1a87ebe799f176e53ebfa6cebadf2b4f4418dfb6cb7b
+  data.tar.gz: 45bec524fc7a727031cf65b98c4b1db41dc8f9a1bfe86696fe0e7e3df7f8a0fa
 SHA512:
-  metadata.gz: dcac3281832905427ad5799b6b1d7c9dab9b13562c19ba8283fa4eaf2ea1b88e7c782714be7cec9546878e26ab223725b0cbc06259bc8e375eff5e407575bb3f
-  data.tar.gz: deef97208222254e9fc4b0279f0dd9045c177354fa1a8dcca3a304d3b2a5d1090e1e7970b5cd9066d7ff917e8020b798d4dd674623285c52041c173fe37d48f5
+  metadata.gz: fdee5e4b8d0e15f835d2b7d3bedb0d735e0b00b9e725fac19ed68c710a751c99279aa9966590cab0fef7c5b88a28cd7cefb80ee923423cc47e49a13ba88ac8f5
+  data.tar.gz: a33c14f6c4927e3cd91c8aabbf2edc35a34582aa5859b5369829721810234edcdad39a2b148d4bff3377177f707bae3e74909d1f8b553cb8ee3682d00e4d58d7

data/ChangeLog

@@ -1,3 +1,24 @@
+Release 0.13.1 - 2020/07/17
+
+	* in_kafka_group: Support ssl_verify_hostname parameter
+	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
+
+Release 0.13.0 - 2020/03/09
+
+	* Accept ruby-kafka v1 or later
+
+Release 0.12.4 - 2020/03/03
+
+	* output: Follow rdkafka log level
+
+Release 0.12.3 - 2020/02/06
+
+	* output: Show warning message for v0.12 plugins
+
+Release 0.12.2 - 2020/01/07
+
+	* input: Refer sasl_over_ssl parameter in plain SASL
+
 Release 0.12.1 - 2019/10/14
 
 	* input: Add time_source parameter to replace use_record_time

data/README.md

@@ -4,8 +4,6 @@
 
 A fluentd plugin to both consume and produce data for Apache Kafka.
 
-TODO: Also, I need to write tests
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -123,7 +121,8 @@ Consume events by kafka consumer group features..
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
       retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
-      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      use_record_time (Deprecated. Use 'time_source record' instead.) <If true, replace event time with contents of 'time' field of fetched record>
+      time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
       # ruby-kafka consumer options
@@ -142,7 +141,7 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 ### Output plugin
 
-This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -157,6 +156,8 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
@@ -180,7 +181,7 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
 
       # ruby-kafka producer options
       idempotent        (bool)    :default => false
-      sasl_over_ssl     (bool)    :default => false
+      sasl_over_ssl     (bool)    :default => true
       max_send_retries  (integer) :default => 1
       required_acks     (integer) :default => -1
       ack_timeout       (integer) :default => nil (Use default of ruby-kafka)
@@ -199,8 +200,6 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
-- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
@@ -277,6 +276,7 @@ The configuration format is jsonpath. It is descibed in https://docs.fluentd.org
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
+Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2` and will be removed in the future.
 
     <match app.**>
       @type kafka_buffered
@@ -293,6 +293,10 @@ This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12.
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
+      exclude_topic_key     (bool)   :default => false
+      exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
@@ -304,7 +308,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12.
 
       # ruby-kafka producer options
       idempotent                   (bool)    :default => false
-      sasl_over_ssl                (bool)    :default => false
+      sasl_over_ssl                (bool)    :default => true
       max_send_retries             (integer) :default => 1
       required_acks                (integer) :default => -1
       ack_timeout                  (integer) :default => nil (Use default of ruby-kafka)
@@ -316,6 +320,11 @@ This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12.
       monitoring_list              (array)   :default => []
     </match>
 
+`kafka_buffered` has two additional parameters:
+
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -350,10 +359,10 @@ This plugin also supports ruby-kafka related parameters. See Buffered output plu
 
 ### rdkafka based output plugin
 
-This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.
 You need to install rdkafka gem.
 
-    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    # rdkafka is C extension library. Need to install development tools like ruby-devel, gcc and etc
     # for v0.12 or later
     $ gem install rdkafka --no-document
     # for v0.11 or earlier
@@ -426,6 +435,18 @@ If you use v0.12, use `rdkafka` instead.
       }
     </match>
 
+## FAQ
+
+### Why fluent-plugin-kafka can't send data to our kafka cluster?
+
+We got lots of similar questions. Almost cases, this problem happens by version mismatch between ruby-kafka and kafka cluster.
+See ruby-kafka README for more details: https://github.com/zendesk/ruby-kafka#compatibility
+
+To avoid the problem, there are 2 approaches:
+
+- Upgrade your kafka cluster to latest version. This is better because recent version is faster and robust.
+- Downgrade ruby-kafka/fluent-plugin-kafka to work with your older kafka.
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.12.1'
+  gem.version       = '0.13.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 0.7.8', '< 0.8.0'
+  gem.add_dependency 'ruby-kafka', '>= 0.7.8', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -39,6 +39,8 @@ class Fluent::KafkaInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -190,7 +192,8 @@ class Fluent::KafkaInput < Fluent::Input
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password,
+                         sasl_over_ssl: @sasl_over_ssl)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
@@ -291,9 +294,9 @@ class Fluent::KafkaInput < Fluent::Input
             record_time = Fluent::Engine.now
           when :record
             if @time_format
-              record_time = @time_parser.parse(record['time'])
+              record_time = @time_parser.parse(record[@record_time_key])
             else
-              record_time = record['time']
+              record_time = record[@record_time_key]
             end
           else
             $log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/in_kafka_group.rb

@@ -29,6 +29,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -166,15 +168,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
+                         sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
+                         ssl_verify_hostname: @ssl_verify_hostname)
     end
 
     @consumer = setup_consumer
@@ -197,7 +201,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def setup_consumer
     consumer = @kafka.consumer(@consumer_opts)
     @topics.each { |topic|
-      consumer.subscribe(topic, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
+      if m = /^\/(.+)\/$/.match(topic)
+        topic_or_regex = Regexp.new(m[1])
+        $log.info "Subscribe to topics matching the regex #{topic}"
+      else
+        topic_or_regex = topic
+        $log.info "Subscribe to topic #{topic}"
+      end
+      consumer.subscribe(topic_or_regex, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
     }
     consumer
   end
@@ -242,9 +253,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
                 record_time = Fluent::Engine.now
               when :record
                 if @time_format
-                  record_time = @time_parser.parse(record['time'].to_s)
+                  record_time = @time_parser.parse(record[@record_time_key].to_s)
                 else
-                  record_time = record['time']
+                  record_time = record[@record_time_key]
                 end
               else
                 log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/out_kafka.rb

@@ -131,6 +131,8 @@ DESC
   def configure(conf)
     super
 
+    log.warn "Support of fluentd v0.12 has ended. Use kafka2 instead. kafka will be an alias of kafka2"
+
     if @zookeeper
       require 'zookeeper'
     else

data/lib/fluent/plugin/out_kafka2.rb

@@ -15,6 +15,7 @@ module Fluent::Plugin
 Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -215,7 +216,11 @@ DESC
     # TODO: optimize write performance
     def write(chunk)
       tag = chunk.metadata.tag
-      topic =  (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       messages = 0
       record_buf = nil

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -159,6 +159,8 @@ DESC
   def configure(conf)
     super
 
+    log.warn "Support of fluentd v0.12 has ended. Use kafka2 instead. kafka_buffered will be an alias of kafka2"
+
     if @zookeeper
       require 'zookeeper'
     else

data/lib/fluent/plugin/out_rdkafka.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -91,8 +92,22 @@ DESC
     super
     log.instance_eval {
       def add(level, &block)
-        if block
+        return unless block
+
+        # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+        case level
+        when Logger::FATAL
+          self.fatal(block.call)
+        when Logger::ERROR
+          self.error(block.call)
+        when Logger::WARN
+          self.warn(block.call)
+        when Logger::INFO
           self.info(block.call)
+        when Logger::DEBUG
+          self.debug(block.call)
+        else
+          self.trace(block.call)
         end
       end
     }

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -32,6 +33,7 @@ Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 Brokers: you can choose to use either brokers or zookeeper.
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -108,8 +110,22 @@ DESC
       super
       log.instance_eval {
         def add(level, &block)
-          if block
+          return unless block
+
+          # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+          case level
+          when Logger::FATAL
+            self.fatal(block.call)
+          when Logger::ERROR
+            self.error(block.call)
+          when Logger::WARN
+            self.warn(block.call)
+          when Logger::INFO
             self.info(block.call)
+          when Logger::DEBUG
+            self.debug(block.call)
+          else
+            self.trace(block.call)
           end
         end
       }
@@ -263,7 +279,11 @@ DESC
 
     def write(chunk)
       tag = chunk.metadata.tag
-      topic = (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       handlers = []
       record_buf = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-15 00:00:00.000000000 Z
+date: 2020-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -54,7 +54,7 @@ dependencies:
         version: 0.7.8
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -64,7 +64,7 @@ dependencies:
         version: 0.7.8
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement