logstash-integration-kafka 11.3.4-java → 11.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb2e5a937c705523e56611e75be5b76453c42b6d5ed241cf86f990251aa673c7
-  data.tar.gz: 379bad770fdc823d19d676041247949f55f51f2f77fe267c7be5fc9f67141053
+  metadata.gz: fa23de31ba8ec88920997f731d28d11e68148b0179b85abeddcfc999f9e96bcf
+  data.tar.gz: bed04421d5654eb1cb3774c60e0a2989be5d79edf537ec1c23ca5c70bedfec82
 SHA512:
-  metadata.gz: 320e5ee1e26113cceb2692c2ec6852f8c9b41adbcba9609e7e917116040c85196c91a0051fcc6a40213e97092c1f3ea4507930dc6b9f7bbf9501dd1c2caaab35
-  data.tar.gz: ab200d48d0d3713f14ab97095d5b56467e79b6c4c22ef23bb32cf26eb7e80f1497565d4b6577d31373bae472bd2955a8dbb59fcd89674c20c3ab2acbe01842bb
+  metadata.gz: 38688f3130feb2ad66bef528007250d98adb28d9d8f3ab1df3817c894dceadbadbd7fa6a62f98b046c97b79089dcaadabda9f05b1663d58249e1f4e37fa845c0
+  data.tar.gz: 618a06ed2dd6c15c05f76e293ac83dd226e9b5d60876f35f58027468a8701468c0dcf4b621fdadb245697211ea7cde9d498af0f7123e0ab0c7299d229cd95c2d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.4.0
+  - Add support for setting Kafka message headers in output plugin [#162](https://github.com/logstash-plugins/logstash-integration-kafka/pull/162)
+
 ## 11.3.4
   - Fix "retries" and "value_serializer" error handling in output plugin (#160) [#160](https://github.com/logstash-plugins/logstash-integration-kafka/pull/160)
 

data/docs/output-kafka.asciidoc

@@ -91,6 +91,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-linger_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_request_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-message_header>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-message_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-metadata_max_age_ms>> |<<number,number>>|No
@@ -275,6 +276,22 @@ to allow other records to be sent so that the sends can be batched together.
 
 The maximum size of a request
 
+[id="plugins-{type}s-{plugin}-message_headers"]
+===== `message_headers`
+
+  * Value type is <<hash,hash>>
+  ** Keys are header names, and must be <<string,string>>
+  ** Values are header values, and must be <<string,string>>
+  ** Values support interpolation from event field values
+  * There is no default value for this setting.
+
+A map of key value pairs, each corresponding to a header name and its value respectively.
+Example:
+[source,ruby]
+----------------------------------
+    message_header => { "event_timestamp" => "%{@timestamp}" }
+----------------------------------
+
 [id="plugins-{type}s-{plugin}-message_key"]
 ===== `message_key` 
 

data/lib/logstash/outputs/kafka.rb

@@ -106,6 +106,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :max_request_size, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The key for the message
   config :message_key, :validate => :string
+  # Headers added to kafka message in the form of key-value pairs
+  config :message_headers, :validate => :hash, :default => {}
   # the timeout setting for initial metadata request to fetch topic metadata.
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
   # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
@@ -204,6 +206,11 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     else
       raise LogStash::ConfigurationError, "'value_serializer' only supports org.apache.kafka.common.serialization.ByteArraySerializer and org.apache.kafka.common.serialization.StringSerializer"
     end
+    @message_headers.each do |key, value|
+      if !key.is_a? String
+        raise LogStash::ConfigurationError, "'message_headers' contains a key that is not a string!"
+      end
+    end
     @producer = create_producer
   end
 
@@ -315,6 +322,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     else
       record = ProducerRecord.new(event.sprintf(@topic_id), event.sprintf(@message_key), serialized_data)
     end
+    @message_headers.each do |key, value|
+      record.headers().add(key, event.sprintf(value).to_java_bytes)
+    end
     prepare(record)
   rescue LogStash::ShutdownSignal
     logger.debug('producer received shutdown signal')

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.3.4'
+  s.version         = '11.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/integration/outputs/kafka_spec.rb

@@ -191,6 +191,25 @@ describe "outputs/kafka", :integration => true do
     end
   end
 
+  context 'when setting message_headers' do
+    let(:num_events) { 10 }
+    let(:test_topic) { 'logstash_integration_topic4' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "message_headers" => {"event_timestamp" => "%{@timestamp}"}})
+      load_kafka_data(config)
+    end
+
+    it 'messages should contain headers' do
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
+        expect(m.headers).to eq({"event_timestamp" => LogStash::Timestamp.at(0).to_s})
+      end
+    end
+  end
+
   context 'setting partitioner' do
     let(:test_topic) { 'logstash_integration_partitioner_topic' }
     let(:partitioner) { nil }

data/spec/unit/outputs/kafka_spec.rb

@@ -60,6 +60,16 @@ describe "outputs/kafka" do
       kafka.multi_receive([event])
     end
 
+    it 'should support field referenced message_headers' do
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("test", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+      expect_any_instance_of(org.apache.kafka.common.header.internals.RecordHeaders).to receive(:add).with("host","172.0.0.1".to_java_bytes).and_call_original
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_headers" => { "host" => "%{host}"}}))
+      kafka.register
+      kafka.multi_receive([event])
+    end
+
     it 'should not raise config error when truststore location is not set and ssl is enabled' do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
       expect(org.apache.kafka.clients.producer.KafkaProducer).to receive(:new)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.3.4
+  version: 11.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement