logstash-integration-kafka 10.7.2-java → 10.7.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ca014c7463762ed6c2f4be1a9c0dce13356980f7ddf244294dd02837862d54f
-  data.tar.gz: 1d68b07fa127636a6bba867ffa0fff6beefd5b1d0a7ba1419eedcea224a97fe5
+  metadata.gz: 1049dfcad573e64aed24bebc6114b0345ce6513e43c2104c54ac8e3ed6602e57
+  data.tar.gz: 41260002d0a2765b4beeac73bdd8d8f100e0b3e058c1e5dbac4f2a460a29a3d0
 SHA512:
-  metadata.gz: 97c32c1cdf91c205a61fefa6ec23544b45cbc66205418df85b2e7bb0c724a2a5672f658b947c086f38808b445d99bc053ee0651eae6b3da0e5cee6065f519c59
-  data.tar.gz: 6f67718001a257aafa339dde326192eef009a1bd74a1c1d4e8b37ca277362f95da3a56100819d9ec972a782e0631a5e4c91013d59efab720ea8e8c39b9d100a4
+  metadata.gz: 744b4621e51c32ef882c7f5521be467c75805c2ba6ee8eaf225567412fe9e6bade35e60ae7fa85b87de50f68a191f5408b0504025e28071e11eb227ab5f428c8
+  data.tar.gz: feb75d5f0da3523e891253213cf52b49460881c125adf2fc82710f96c0feaa3eb4630468bbec7f1182c6a52e44b060362b5dd946b6194c066821a3de4c6e1314

data/CHANGELOG.md

@@ -1,8 +1,11 @@
+## 10.7.3
+  - Changed `decorate_events` to add also Kafka headers [#78](https://github.com/logstash-plugins/logstash-integration-kafka/pull/78)
+
 ## 10.7.2
   - Update Jersey dependency to version 2.33 [#75](https://github.com/logstash-plugins/logstash-integration-kafka/pull/75)
 
 ## 10.7.1
-  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/issue/71)
+  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/pull/71)
   
 ## 10.7.0
   - Switched use from Faraday to Manticore as HTTP client library to access Schema Registry service 

data/docs/input-kafka.asciidoc

@@ -73,7 +73,7 @@ either when the record was created (default) or when it was received by the
 broker. See more about property log.message.timestamp.type at
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
-Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
+Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`).
 
 Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be 
 inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`.
@@ -99,7 +99,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-consumer_threads>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-decorate_events>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-decorate_events>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-enable_auto_commit>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-exclude_internal_topics>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-fetch_max_bytes>> |<<number,number>>|No
@@ -246,10 +246,16 @@ balance — more threads than partitions means that some threads will be idl
 [id="plugins-{type}s-{plugin}-decorate_events"]
 ===== `decorate_events` 
 
-  * Value type is <<boolean,boolean>>
-  * Default value is `false`
-
-Option to add Kafka metadata like topic, message size to the event.
+  * Value type is <<string,string>>
+  * Accepted values are:
+    - `none`: no metadata is added
+    - `basic`: record's attributes are added
+    - `extended`: record's attributes, headers are added
+    - `false`: deprecated alias for `none`
+    - `true`: deprecated alias for `basic`
+  * Default value is `none`
+
+Option to add Kafka metadata like topic, message size and header key values to the event.
 This will add a field named `kafka` to the logstash event containing the following attributes:
 
 * `topic`: The topic this message is associated with

data/lib/logstash/inputs/kafka.rb

@@ -8,6 +8,7 @@ require 'manticore'
 require "json"
 require "logstash/json"
 require_relative '../plugin_mixins/common'
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -58,6 +59,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   include LogStash::PluginMixins::KafkaSupport
   include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
 
@@ -233,22 +235,49 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :sasl_jaas_config, :validate => :string
   # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
   config :kerberos_config, :validate => :path
-  # Option to add Kafka metadata like topic, message size to the event.
-  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  # Option to add Kafka metadata like topic, message size and header key values to the event.
+  # With `basic` this will add a field named `kafka` to the logstash event containing the following attributes:
   #   `topic`: The topic this message is associated with
   #   `consumer_group`: The consumer group used to read in this event
   #   `partition`: The partition this message is associated with
   #   `offset`: The offset from the partition this message is associated with
   #   `key`: A ByteBuffer containing the message key
   #   `timestamp`: The timestamp of this message
-  config :decorate_events, :validate => :boolean, :default => false
+  # While with `extended` it adds also all the key values present in the Kafka header if the key is valid UTF-8 else
+  # silently skip it.
+  config :decorate_events, :validate => %w(none basic extended false true), :default => "none"
+
+  attr_reader :metadata_mode
 
   public
   def register
     @runner_threads = []
+    @metadata_mode = extract_metadata_level(@decorate_events)
     check_schema_registry_parameters
   end
 
+  METADATA_NONE     = Set[].freeze
+  METADATA_BASIC    = Set[:record_props].freeze
+  METADATA_EXTENDED = Set[:record_props, :headers].freeze
+  METADATA_DEPRECATION_MAP = { 'true' => 'basic', 'false' => 'none' }
+
+  private
+  def extract_metadata_level(decorate_events_setting)
+    metadata_enabled = decorate_events_setting
+
+    if METADATA_DEPRECATION_MAP.include?(metadata_enabled)
+      canonical_value = METADATA_DEPRECATION_MAP[metadata_enabled]
+      deprecation_logger.deprecated("Deprecated value `#{decorate_events_setting}` for `decorate_events` option; use `#{canonical_value}` instead.")
+      metadata_enabled = canonical_value
+    end
+
+    case metadata_enabled
+    when 'none'     then METADATA_NONE
+    when 'basic'    then METADATA_BASIC
+    when 'extended' then METADATA_EXTENDED
+    end
+  end
+
   public
   def run(logstash_queue)
     @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
@@ -292,7 +321,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                 end
                 event.remove("message")
               end
-              if @decorate_events
+              if @metadata_mode.include?(:record_props)
                 event.set("[@metadata][kafka][topic]", record.topic)
                 event.set("[@metadata][kafka][consumer_group]", @group_id)
                 event.set("[@metadata][kafka][partition]", record.partition)
@@ -300,6 +329,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                 event.set("[@metadata][kafka][key]", record.key)
                 event.set("[@metadata][kafka][timestamp]", record.timestamp)
               end
+              if @metadata_mode.include?(:headers)
+                record.headers.each do |header|
+                  s = String.from_java_bytes(header.value)
+                  s.force_encoding(Encoding::UTF_8)
+                  if s.valid_encoding?
+                    event.set("[@metadata][kafka][headers]["+header.key+"]", s)
+                  end
+                end
+              end
               logstash_queue << event
             end
           end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.2'
+  s.version         = '10.7.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
   s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'

data/spec/integration/inputs/kafka_spec.rb

@@ -36,7 +36,15 @@ describe "inputs/kafka", :integration => true do
   end
   let(:decorate_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
-      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
+  end
+  let(:decorate_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
+  end
+  let(:decorate_bad_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
@@ -45,6 +53,35 @@ describe "inputs/kafka", :integration => true do
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
+  before(:all) do
+    # Prepare message with headers with valid UTF-8 chars
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", "John ανδρεα €".to_java_bytes)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers", 0, "key", "value", [header])
+    send_message(record)
+
+    # Prepare message with headers with invalid UTF-8 chars
+    invalid = "日本".encode('Shift_JIS').force_encoding(Encoding::UTF_8).to_java_bytes
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", invalid)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers_badly", 0, "key", "value", [header])
+
+    send_message(record)
+  end
+
+  def send_message(record)
+    props = java.util.Properties.new
+    kafka = org.apache.kafka.clients.producer.ProducerConfig
+    props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+    props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+
+    producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+
+    producer.send(record)
+    producer.close
+  end
+
   describe "#kafka-topics" do
 
     it "should consume all messages from plain 3-partition topic" do
@@ -74,7 +111,7 @@ describe "inputs/kafka", :integration => true do
 
   context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      total_events = num_events * 3
+      total_events = num_events * 3 + 2
       queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
       expect(queue.length).to eq(total_events)
     end
@@ -91,6 +128,31 @@ describe "inputs/kafka", :integration => true do
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
       end
     end
+
+    it "should show the right topic and group name in and kafka headers decorated kafka section" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+        expect(event.get("[@metadata][kafka][headers][name]")).to eq("John ανδρεα €")
+      end
+    end
+
+    it "should skip headers not encoded in UTF-8" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_bad_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers_badly")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+
+        expect(event.include?("[@metadata][kafka][headers][name]")).to eq(false)
+      end
+    end
   end
 
   context "#kafka-offset-commit" do
@@ -129,6 +191,7 @@ private
 
 def consume_messages(config, queue: Queue.new, timeout:, event_count:)
   kafka_input = LogStash::Inputs::Kafka.new(config)
+  kafka_input.register
   t = Thread.new { kafka_input.run(queue) }
   begin
     t.run

data/spec/unit/inputs/kafka_spec.rb

@@ -38,18 +38,35 @@ describe LogStash::Inputs::Kafka do
   end
 
   context "register parameter verification" do
-    let(:config) do
-      { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-    end
+    context "schema_registry_url" do
+      let(:config) do
+        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
+      end
 
-    it "schema_registry_url conflict with value_deserializer_class should fail" do
-      config['value_deserializer_class'] = 'my.fantasy.Deserializer'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      it "conflict with value_deserializer_class should fail" do
+        config['value_deserializer_class'] = 'my.fantasy.Deserializer'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      end
+
+      it "conflict with topics_pattern should fail" do
+        config['topics_pattern'] = 'topic_.*'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+      end
     end
 
-    it "schema_registry_url conflict with topics_pattern should fail" do
-      config['topics_pattern'] = 'topic_.*'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+    context "decorate_events" do
+      let(:config) { { 'decorate_events' => 'extended'} }
+
+      it "should raise error for invalid value" do
+        config['decorate_events'] = 'avoid'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+      end
+
+      it "should map old true boolean value to :record_props mode" do
+        config['decorate_events'] = "true"
+        subject.register
+        expect(subject.metadata_mode).to include(:record_props)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.2
+  version: 10.7.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-05 00:00:00.000000000 Z
+date: 2021-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: