fluent-plugin-kafka 0.12.3 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5c5fbae81fab15c17ce05013a3a26d924b91cb58c5be753ffdd2c87733667c6
-  data.tar.gz: fc40bae1a4d1c391c9da98c52e442065f2291e7899a8ab20eb4067181efd6a68
+  metadata.gz: 27a59b5835dff5d64dcf78bd5d3bf945341c9b734de476b16f9217afa2839a22
+  data.tar.gz: fdac125fa11e88712059f0f0794ca199ef248d562b1e1aad389e6b0dace9c777
 SHA512:
-  metadata.gz: f5df728c0e3516ab4548ad139972fa79729dd5f25d4c5f94423217110f668f15d65fb2d191c6231f446775874ffd2e6b834ad678b8587c1bf47cede4248587f4
-  data.tar.gz: 8a88fc87aef958ec0c43400a6a67e25a01475a607e9d4f38e4da39ac5a926cc2a29199fc911c1258a619a8ca844a2e32d0e2755e8f1c029056fb3e1fb66794d7
+  metadata.gz: faf2abd472b6af6b010409750d6b0e3483ba8748af8930c4dad05d6e6b4d9aca5dc7c55d29b2e4ed3b092b12612f0d19e46b9878463ecbe417e417d7c3ee522b
+  data.tar.gz: f51803596ea03e0f6dfc9f83abaf070179136f497dc89ad48eed0582c8d580ad88e105f5f4e075096421f6dc0acf8bf1e58fd75738e7c49636acca01ec347a46

data/ChangeLog

@@ -1,3 +1,25 @@
+Release 0.14.1 - 2020/08/11
+
+	* kafka_producer_ext: Fix regression by v0.14.0 changes
+
+Release 0.14.0 - 2020/08/07
+
+	* Update ruby-kafka dependency to v1.2.0 or later. Check https://github.com/zendesk/ruby-kafka#compatibility
+	* kafka_producer_ext: Follow Paritioner API change
+
+Release 0.13.1 - 2020/07/17
+
+	* in_kafka_group: Support ssl_verify_hostname parameter
+	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
+
+Release 0.13.0 - 2020/03/09
+
+	* Accept ruby-kafka v1 or later
+
+Release 0.12.4 - 2020/03/03
+
+	* output: Follow rdkafka log level
+
 Release 0.12.3 - 2020/02/06
 
 	* output: Show warning message for v0.12 plugins

data/README.md

@@ -121,7 +121,8 @@ Consume events by kafka consumer group features..
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
       retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
-      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      use_record_time (Deprecated. Use 'time_source record' instead.) <If true, replace event time with contents of 'time' field of fetched record>
+      time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
       # ruby-kafka consumer options
@@ -140,7 +141,7 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 ### Output plugin
 
-This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -155,6 +156,8 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
@@ -197,8 +200,6 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
-- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
@@ -292,6 +293,10 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
+      exclude_topic_key     (bool)   :default => false
+      exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
@@ -315,6 +320,11 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       monitoring_list              (array)   :default => []
     </match>
 
+`kafka_buffered` has two additional parameters:
+
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -349,10 +359,10 @@ This plugin also supports ruby-kafka related parameters. See Buffered output plu
 
 ### rdkafka based output plugin
 
-This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.
 You need to install rdkafka gem.
 
-    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    # rdkafka is C extension library. Need to install development tools like ruby-devel, gcc and etc
     # for v0.12 or later
     $ gem install rdkafka --no-document
     # for v0.11 or earlier
@@ -434,7 +444,7 @@ See ruby-kafka README for more details: https://github.com/zendesk/ruby-kafka#co
 
 To avoid the problem, there are 2 approaches:
 
-- Upgrade your kafka cluster to latest version. This is better becase recent version is faster and robust.
+- Upgrade your kafka cluster to latest version. This is better because recent version is faster and robust.
 - Downgrade ruby-kafka/fluent-plugin-kafka to work with your older kafka.
 
 ## Contributing

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.12.3'
+  gem.version       = '0.14.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 0.7.8', '< 0.8.0'
+  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -39,6 +39,8 @@ class Fluent::KafkaInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -292,9 +294,9 @@ class Fluent::KafkaInput < Fluent::Input
             record_time = Fluent::Engine.now
           when :record
             if @time_format
-              record_time = @time_parser.parse(record['time'])
+              record_time = @time_parser.parse(record[@record_time_key])
             else
-              record_time = record['time']
+              record_time = record[@record_time_key]
             end
           else
             $log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/in_kafka_group.rb

@@ -29,6 +29,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -166,16 +168,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
-                         sasl_over_ssl: @sasl_over_ssl)
+                         sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
+                         ssl_verify_hostname: @ssl_verify_hostname)
     end
 
     @consumer = setup_consumer
@@ -198,7 +201,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def setup_consumer
     consumer = @kafka.consumer(@consumer_opts)
     @topics.each { |topic|
-      consumer.subscribe(topic, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
+      if m = /^\/(.+)\/$/.match(topic)
+        topic_or_regex = Regexp.new(m[1])
+        $log.info "Subscribe to topics matching the regex #{topic}"
+      else
+        topic_or_regex = topic
+        $log.info "Subscribe to topic #{topic}"
+      end
+      consumer.subscribe(topic_or_regex, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
     }
     consumer
   end
@@ -243,9 +253,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
                 record_time = Fluent::Engine.now
               when :record
                 if @time_format
-                  record_time = @time_parser.parse(record['time'].to_s)
+                  record_time = @time_parser.parse(record[@record_time_key].to_s)
                 else
-                  record_time = record['time']
+                  record_time = record[@record_time_key]
                 end
               else
                 log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -69,12 +69,13 @@ module Kafka
         retry_backoff: retry_backoff,
         max_buffer_size: max_buffer_size,
         max_buffer_bytesize: max_buffer_bytesize,
+        partitioner: @partitioner,
       )
     end
   end
 
   class TopicProducer
-    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
+    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:, partitioner:)
       @cluster = cluster
       @transaction_manager = transaction_manager
       @logger = logger
@@ -86,6 +87,7 @@ module Kafka
       @max_buffer_size = max_buffer_size
       @max_buffer_bytesize = max_buffer_bytesize
       @compressor = compressor
+      @partitioner = partitioner
 
       @topic = topic
       @cluster.add_target_topics(Set.new([topic]))
@@ -250,7 +252,7 @@ module Kafka
 
         begin
           if partition.nil?
-            partition = Partitioner.partition_for_key(partition_count, message)
+            partition = @partitioner.call(partition_count, message)
           end
 
           @buffer.write(

data/lib/fluent/plugin/out_kafka2.rb

@@ -15,6 +15,7 @@ module Fluent::Plugin
 Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -215,7 +216,11 @@ DESC
     # TODO: optimize write performance
     def write(chunk)
       tag = chunk.metadata.tag
-      topic =  (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       messages = 0
       record_buf = nil

data/lib/fluent/plugin/out_rdkafka.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -91,8 +92,22 @@ DESC
     super
     log.instance_eval {
       def add(level, &block)
-        if block
+        return unless block
+
+        # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+        case level
+        when Logger::FATAL
+          self.fatal(block.call)
+        when Logger::ERROR
+          self.error(block.call)
+        when Logger::WARN
+          self.warn(block.call)
+        when Logger::INFO
           self.info(block.call)
+        when Logger::DEBUG
+          self.debug(block.call)
+        else
+          self.trace(block.call)
         end
       end
     }

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -32,6 +33,7 @@ Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 Brokers: you can choose to use either brokers or zookeeper.
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -108,8 +110,22 @@ DESC
       super
       log.instance_eval {
         def add(level, &block)
-          if block
+          return unless block
+
+          # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+          case level
+          when Logger::FATAL
+            self.fatal(block.call)
+          when Logger::ERROR
+            self.error(block.call)
+          when Logger::WARN
+            self.warn(block.call)
+          when Logger::INFO
             self.info(block.call)
+          when Logger::DEBUG
+            self.debug(block.call)
+          else
+            self.trace(block.call)
           end
         end
       }
@@ -263,7 +279,11 @@ DESC
 
     def write(chunk)
       tag = chunk.metadata.tag
-      topic = (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       handlers = []
       record_buf = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.12.3
+  version: 0.14.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-06 00:00:00.000000000 Z
+date: 2020-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,20 +51,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement