fluent-plugin-kafka 0.12.2 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9dd9e59977840b92e4db04a067c6a5af66c63b982f808511b1118d33a37604a5
-  data.tar.gz: fb6445978009096d7461b6da7605e0e51d0402a43c56690a8c0912844555af3b
+  metadata.gz: 0ceff7f76a27f8be74e0f6ffd01cf793a951a78aa14c73c2758472a08d5d6b7e
+  data.tar.gz: c95a162564fea2b7abbedc1032151c87d5c5938c56c6fa6d871eb678306d4e7e
 SHA512:
-  metadata.gz: 5ccfca260f9a688203bb44a64ea9ff07abacd1ce0089aed664bfb67e5001b24e4e5a848c373273f0fed3727fa4dae125c959e27c7628b34a3db12efe030c5ad8
-  data.tar.gz: b6a79e20ed5e19434161864f9dfd96df3d5a140fdbadd6242b88992a1b9327b24089b519275f9c7e2fc256d824430818068e6613317b22e3a31a0ecaf3e0e7f7
+  metadata.gz: a78e16a0b4e0995699f5d0d5e7010eed8ecfa52aeff647e83c9f885299396d184b8e2b656fd4626519f1e93297210a0b6ae59e6ca1dc5c066cbc6b644e83e36f
+  data.tar.gz: 98cfcb42d5b225861d90d7adb3f2a1eaba93a57c7f08ed142e767f4a971c15359e8c866e73f554405021bb037ea1935f0d85d5a0b08e999e2181e31421f99ba4

data/ChangeLog

@@ -1,3 +1,25 @@
+Release 0.14.0 - 2020/08/07
+
+	* Update ruby-kafka dependency to v1.2.0 or later. Check https://github.com/zendesk/ruby-kafka#compatibility
+	* kafka_producer_ext: Follow Paritioner API change
+
+Release 0.13.1 - 2020/07/17
+
+	* in_kafka_group: Support ssl_verify_hostname parameter
+	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
+
+Release 0.13.0 - 2020/03/09
+
+	* Accept ruby-kafka v1 or later
+
+Release 0.12.4 - 2020/03/03
+
+	* output: Follow rdkafka log level
+
+Release 0.12.3 - 2020/02/06
+
+	* output: Show warning message for v0.12 plugins
+
 Release 0.12.2 - 2020/01/07
 
 	* input: Refer sasl_over_ssl parameter in plain SASL

data/README.md

@@ -121,7 +121,8 @@ Consume events by kafka consumer group features..
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
       retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
-      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      use_record_time (Deprecated. Use 'time_source record' instead.) <If true, replace event time with contents of 'time' field of fetched record>
+      time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
       # ruby-kafka consumer options
@@ -140,7 +141,7 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 ### Output plugin
 
-This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -155,6 +156,8 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
@@ -178,7 +181,7 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
 
       # ruby-kafka producer options
       idempotent        (bool)    :default => false
-      sasl_over_ssl     (bool)    :default => false
+      sasl_over_ssl     (bool)    :default => true
       max_send_retries  (integer) :default => 1
       required_acks     (integer) :default => -1
       ack_timeout       (integer) :default => nil (Use default of ruby-kafka)
@@ -197,8 +200,6 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
-- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
@@ -275,6 +276,7 @@ The configuration format is jsonpath. It is descibed in https://docs.fluentd.org
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
+Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2` and will be removed in the future.
 
     <match app.**>
       @type kafka_buffered
@@ -291,6 +293,10 @@ This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12.
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
+      exclude_topic_key     (bool)   :default => false
+      exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
@@ -302,7 +308,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12.
 
       # ruby-kafka producer options
       idempotent                   (bool)    :default => false
-      sasl_over_ssl                (bool)    :default => false
+      sasl_over_ssl                (bool)    :default => true
       max_send_retries             (integer) :default => 1
       required_acks                (integer) :default => -1
       ack_timeout                  (integer) :default => nil (Use default of ruby-kafka)
@@ -314,6 +320,11 @@ This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12.
       monitoring_list              (array)   :default => []
     </match>
 
+`kafka_buffered` has two additional parameters:
+
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -348,10 +359,10 @@ This plugin also supports ruby-kafka related parameters. See Buffered output plu
 
 ### rdkafka based output plugin
 
-This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.
 You need to install rdkafka gem.
 
-    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    # rdkafka is C extension library. Need to install development tools like ruby-devel, gcc and etc
     # for v0.12 or later
     $ gem install rdkafka --no-document
     # for v0.11 or earlier
@@ -433,7 +444,7 @@ See ruby-kafka README for more details: https://github.com/zendesk/ruby-kafka#co
 
 To avoid the problem, there are 2 approaches:
 
-- Upgrade your kafka cluster to latest version. This is better becase recent version is faster and robust.
+- Upgrade your kafka cluster to latest version. This is better because recent version is faster and robust.
 - Downgrade ruby-kafka/fluent-plugin-kafka to work with your older kafka.
 
 ## Contributing

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.12.2'
+  gem.version       = '0.14.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 0.7.8', '< 0.8.0'
+  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -39,6 +39,8 @@ class Fluent::KafkaInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -292,9 +294,9 @@ class Fluent::KafkaInput < Fluent::Input
             record_time = Fluent::Engine.now
           when :record
             if @time_format
-              record_time = @time_parser.parse(record['time'])
+              record_time = @time_parser.parse(record[@record_time_key])
             else
-              record_time = record['time']
+              record_time = record[@record_time_key]
             end
           else
             $log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/in_kafka_group.rb

@@ -29,6 +29,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -166,16 +168,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
-                         sasl_over_ssl: @sasl_over_ssl)
+                         sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
+                         ssl_verify_hostname: @ssl_verify_hostname)
     end
 
     @consumer = setup_consumer
@@ -198,7 +201,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def setup_consumer
     consumer = @kafka.consumer(@consumer_opts)
     @topics.each { |topic|
-      consumer.subscribe(topic, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
+      if m = /^\/(.+)\/$/.match(topic)
+        topic_or_regex = Regexp.new(m[1])
+        $log.info "Subscribe to topics matching the regex #{topic}"
+      else
+        topic_or_regex = topic
+        $log.info "Subscribe to topic #{topic}"
+      end
+      consumer.subscribe(topic_or_regex, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
     }
     consumer
   end
@@ -243,9 +253,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
                 record_time = Fluent::Engine.now
               when :record
                 if @time_format
-                  record_time = @time_parser.parse(record['time'].to_s)
+                  record_time = @time_parser.parse(record[@record_time_key].to_s)
                 else
-                  record_time = record['time']
+                  record_time = record[@record_time_key]
                 end
               else
                 log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -250,7 +250,7 @@ module Kafka
 
         begin
           if partition.nil?
-            partition = Partitioner.partition_for_key(partition_count, message)
+            partition = Partitioner.call(partition_count, message)
           end
 
           @buffer.write(

data/lib/fluent/plugin/out_kafka.rb

@@ -131,6 +131,8 @@ DESC
   def configure(conf)
     super
 
+    log.warn "Support of fluentd v0.12 has ended. Use kafka2 instead. kafka will be an alias of kafka2"
+
     if @zookeeper
       require 'zookeeper'
     else

data/lib/fluent/plugin/out_kafka2.rb

@@ -15,6 +15,7 @@ module Fluent::Plugin
 Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -215,7 +216,11 @@ DESC
     # TODO: optimize write performance
     def write(chunk)
       tag = chunk.metadata.tag
-      topic =  (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       messages = 0
       record_buf = nil

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -159,6 +159,8 @@ DESC
   def configure(conf)
     super
 
+    log.warn "Support of fluentd v0.12 has ended. Use kafka2 instead. kafka_buffered will be an alias of kafka2"
+
     if @zookeeper
       require 'zookeeper'
     else

data/lib/fluent/plugin/out_rdkafka.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -91,8 +92,22 @@ DESC
     super
     log.instance_eval {
       def add(level, &block)
-        if block
+        return unless block
+
+        # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+        case level
+        when Logger::FATAL
+          self.fatal(block.call)
+        when Logger::ERROR
+          self.error(block.call)
+        when Logger::WARN
+          self.warn(block.call)
+        when Logger::INFO
           self.info(block.call)
+        when Logger::DEBUG
+          self.debug(block.call)
+        else
+          self.trace(block.call)
         end
       end
     }

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -32,6 +33,7 @@ Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 Brokers: you can choose to use either brokers or zookeeper.
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -108,8 +110,22 @@ DESC
       super
       log.instance_eval {
         def add(level, &block)
-          if block
+          return unless block
+
+          # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+          case level
+          when Logger::FATAL
+            self.fatal(block.call)
+          when Logger::ERROR
+            self.error(block.call)
+          when Logger::WARN
+            self.warn(block.call)
+          when Logger::INFO
             self.info(block.call)
+          when Logger::DEBUG
+            self.debug(block.call)
+          else
+            self.trace(block.call)
           end
         end
       }
@@ -263,7 +279,11 @@ DESC
 
     def write(chunk)
       tag = chunk.metadata.tag
-      topic = (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       handlers = []
       record_buf = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.12.2
+  version: 0.14.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-07 00:00:00.000000000 Z
+date: 2020-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,20 +51,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement