logstash-integration-kafka 11.3.3-java → 11.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b779dc371104b0d12e9c0653c3503c316da6c283f240a48d9e93af8aab1a6657
-  data.tar.gz: 5cc97ac62bf631856947cab6f408bff40f39d77b038fd169445cca102a54beb2
+  metadata.gz: fa23de31ba8ec88920997f731d28d11e68148b0179b85abeddcfc999f9e96bcf
+  data.tar.gz: bed04421d5654eb1cb3774c60e0a2989be5d79edf537ec1c23ca5c70bedfec82
 SHA512:
-  metadata.gz: e6f4198f56a1f964db493573fa8b66a89f6c869885b43b964e43c947f1e81906eac078be550d8d1e32a59f1f5d7b89428adba125dfc4324fbb7fc0075382b3dd
-  data.tar.gz: f27c13d89ebe188e952177dbab113cdc9748b40c854d044a51be94ca69ec636627256e79e403bfac4e32300affbd1176381accc40cb7ee3c5740b51127bfffca
+  metadata.gz: 38688f3130feb2ad66bef528007250d98adb28d9d8f3ab1df3817c894dceadbadbd7fa6a62f98b046c97b79089dcaadabda9f05b1663d58249e1f4e37fa845c0
+  data.tar.gz: 618a06ed2dd6c15c05f76e293ac83dd226e9b5d60876f35f58027468a8701468c0dcf4b621fdadb245697211ea7cde9d498af0f7123e0ab0c7299d229cd95c2d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.4.0
+  - Add support for setting Kafka message headers in output plugin [#162](https://github.com/logstash-plugins/logstash-integration-kafka/pull/162)
+
+## 11.3.4
+  - Fix "retries" and "value_serializer" error handling in output plugin (#160) [#160](https://github.com/logstash-plugins/logstash-integration-kafka/pull/160)
+
 ## 11.3.3
   - Fix "Can't modify frozen string" error when record value is `nil` (tombstones) [#155](https://github.com/logstash-plugins/logstash-integration-kafka/pull/155)
 

data/docs/output-kafka.asciidoc

@@ -91,6 +91,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-linger_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_request_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-message_header>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-message_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-metadata_max_age_ms>> |<<number,number>>|No
@@ -184,7 +185,7 @@ resolved and expanded into a list of canonical names.
 [NOTE]
 ====
 Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed.
-If explicitly configured it fallbacks to `use_all_dns_ips`.
+If not explicitly configured it defaults to `use_all_dns_ips`.
 ====
 
 [id="plugins-{type}s-{plugin}-client_id"]
@@ -275,6 +276,22 @@ to allow other records to be sent so that the sends can be batched together.
 
 The maximum size of a request
 
+[id="plugins-{type}s-{plugin}-message_headers"]
+===== `message_headers`
+
+  * Value type is <<hash,hash>>
+  ** Keys are header names, and must be <<string,string>>
+  ** Values are header values, and must be <<string,string>>
+  ** Values support interpolation from event field values
+  * There is no default value for this setting.
+
+A map of key value pairs, each corresponding to a header name and its value respectively.
+Example:
+[source,ruby]
+----------------------------------
+    message_header => { "event_timestamp" => "%{@timestamp}" }
+----------------------------------
+
 [id="plugins-{type}s-{plugin}-message_key"]
 ===== `message_key` 
 
@@ -348,7 +365,7 @@ retries are exhausted.
   * There is no default value for this setting.
 
 The default retry behavior is to retry until successful. To prevent data loss,
-the use of this setting is discouraged.
+changing this setting is discouraged.
 
 If you choose to set `retries`, a value greater than zero will cause the
 client to only retry a fixed number of times. This will result in data loss

data/lib/logstash/outputs/kafka.rb

@@ -106,6 +106,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :max_request_size, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The key for the message
   config :message_key, :validate => :string
+  # Headers added to kafka message in the form of key-value pairs
+  config :message_headers, :validate => :hash, :default => {}
   # the timeout setting for initial metadata request to fetch topic metadata.
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
   # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
@@ -185,7 +187,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     if !@retries.nil? 
       if @retries < 0
-        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
+        raise LogStash::ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
       end
 
       logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
@@ -193,7 +195,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     reassign_dns_lookup
 
-    @producer = create_producer
     if value_serializer == 'org.apache.kafka.common.serialization.StringSerializer'
       @codec.on_event do |event, data|
         write_to_kafka(event, data)
@@ -203,8 +204,14 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         write_to_kafka(event, data.to_java_bytes)
       end
     else
-      raise ConfigurationError, "'value_serializer' only supports org.apache.kafka.common.serialization.ByteArraySerializer and org.apache.kafka.common.serialization.StringSerializer" 
+      raise LogStash::ConfigurationError, "'value_serializer' only supports org.apache.kafka.common.serialization.ByteArraySerializer and org.apache.kafka.common.serialization.StringSerializer"
+    end
+    @message_headers.each do |key, value|
+      if !key.is_a? String
+        raise LogStash::ConfigurationError, "'message_headers' contains a key that is not a string!"
+      end
     end
+    @producer = create_producer
   end
 
   def prepare(record)
@@ -315,6 +322,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     else
       record = ProducerRecord.new(event.sprintf(@topic_id), event.sprintf(@message_key), serialized_data)
     end
+    @message_headers.each do |key, value|
+      record.headers().add(key, event.sprintf(value).to_java_bytes)
+    end
     prepare(record)
   rescue LogStash::ShutdownSignal
     logger.debug('producer received shutdown signal')

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.3.3'
+  s.version         = '11.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/integration/outputs/kafka_spec.rb

@@ -191,6 +191,25 @@ describe "outputs/kafka", :integration => true do
     end
   end
 
+  context 'when setting message_headers' do
+    let(:num_events) { 10 }
+    let(:test_topic) { 'logstash_integration_topic4' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "message_headers" => {"event_timestamp" => "%{@timestamp}"}})
+      load_kafka_data(config)
+    end
+
+    it 'messages should contain headers' do
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
+        expect(m.headers).to eq({"event_timestamp" => LogStash::Timestamp.at(0).to_s})
+      end
+    end
+  end
+
   context 'setting partitioner' do
     let(:test_topic) { 'logstash_integration_partitioner_topic' }
     let(:partitioner) { nil }

data/spec/unit/outputs/kafka_spec.rb

@@ -60,6 +60,16 @@ describe "outputs/kafka" do
       kafka.multi_receive([event])
     end
 
+    it 'should support field referenced message_headers' do
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("test", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+      expect_any_instance_of(org.apache.kafka.common.header.internals.RecordHeaders).to receive(:add).with("host","172.0.0.1".to_java_bytes).and_call_original
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_headers" => { "host" => "%{host}"}}))
+      kafka.register
+      kafka.multi_receive([event])
+    end
+
     it 'should not raise config error when truststore location is not set and ssl is enabled' do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
       expect(org.apache.kafka.clients.producer.KafkaProducer).to receive(:new)
@@ -221,6 +231,26 @@ describe "outputs/kafka" do
         kafka.multi_receive([event])
       end
     end
+    context 'when retries is -1' do
+      let(:retries) { -1 }
+
+      it "should raise a Configuration error" do
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        expect { kafka.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
+  describe "value_serializer" do
+    let(:output) { LogStash::Plugin.lookup("output", "kafka").new(config) }
+
+    context "when a random string is set" do
+      let(:config) { { "topic_id" => "random", "value_serializer" => "test_string" } }
+
+      it "raises a ConfigurationError" do
+        expect { output.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
   end
 
   context 'when ssl endpoint identification disabled' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.3.3
+  version: 11.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-09 00:00:00.000000000 Z
+date: 2024-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement