logstash-integration-kafka 11.3.2-java → 11.3.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd3536a455fdf7a64f5882e81c0648022d572660d4dc45841249374c1cd7e406
-  data.tar.gz: 26281f8584e29961625c8e385cfd599fd43444cbedd84d191b5542f603d00701
+  metadata.gz: fb2e5a937c705523e56611e75be5b76453c42b6d5ed241cf86f990251aa673c7
+  data.tar.gz: 379bad770fdc823d19d676041247949f55f51f2f77fe267c7be5fc9f67141053
 SHA512:
-  metadata.gz: 226f3c894edb182246d36bc283f7fc8d5d8471d9d411b98054a3770f7e414ca321ddbe401012c53ff5bbf4fd74da0340193153ebf7204a74e03802dc8c3df8ad
-  data.tar.gz: 12a5454b826df30697e4f505e96893687202b8cd17536599a5832a518653274218a278d0139c3150f7c48806e823251f91f3b583ee9b60f828104b5e052fea1b
+  metadata.gz: 320e5ee1e26113cceb2692c2ec6852f8c9b41adbcba9609e7e917116040c85196c91a0051fcc6a40213e97092c1f3ea4507930dc6b9f7bbf9501dd1c2caaab35
+  data.tar.gz: ab200d48d0d3713f14ab97095d5b56467e79b6c4c22ef23bb32cf26eb7e80f1497565d4b6577d31373bae472bd2955a8dbb59fcd89674c20c3ab2acbe01842bb

data/CHANGELOG.md

@@ -1,5 +1,11 @@
+## 11.3.4
+  - Fix "retries" and "value_serializer" error handling in output plugin (#160) [#160](https://github.com/logstash-plugins/logstash-integration-kafka/pull/160)
+
+## 11.3.3
+  - Fix "Can't modify frozen string" error when record value is `nil` (tombstones) [#155](https://github.com/logstash-plugins/logstash-integration-kafka/pull/155)
+
 ## 11.3.2
-  - Fix: update Avro library [#150](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/150)
+  - Fix: update Avro library [#150](https://github.com/logstash-plugins/logstash-integration-kafka/pull/150)
 
 ## 11.3.1
   - Fix: update snappy dependency [#148](https://github.com/logstash-plugins/logstash-integration-kafka/pull/148)

data/docs/output-kafka.asciidoc

@@ -184,7 +184,7 @@ resolved and expanded into a list of canonical names.
 [NOTE]
 ====
 Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed.
-If explicitly configured it fallbacks to `use_all_dns_ips`.
+If not explicitly configured it defaults to `use_all_dns_ips`.
 ====
 
 [id="plugins-{type}s-{plugin}-client_id"]
@@ -348,7 +348,7 @@ retries are exhausted.
   * There is no default value for this setting.
 
 The default retry behavior is to retry until successful. To prevent data loss,
-the use of this setting is discouraged.
+changing this setting is discouraged.
 
 If you choose to set `retries`, a value greater than zero will cause the
 client to only retry a fixed number of times. This will result in data loss

data/lib/logstash/inputs/kafka.rb

@@ -356,7 +356,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end
 
   def handle_record(record, codec_instance, queue)
-    codec_instance.decode(record.value.to_s) do |event|
+    # use + since .to_s on nil/boolean returns a frozen string since ruby 2.7
+    codec_instance.decode(+record.value.to_s) do |event|
       decorate(event)
       maybe_set_metadata(event, record)
       queue << event

data/lib/logstash/outputs/kafka.rb

@@ -185,7 +185,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     if !@retries.nil? 
       if @retries < 0
-        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
+        raise LogStash::ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
       end
 
       logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
@@ -193,7 +193,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     reassign_dns_lookup
 
-    @producer = create_producer
     if value_serializer == 'org.apache.kafka.common.serialization.StringSerializer'
       @codec.on_event do |event, data|
         write_to_kafka(event, data)
@@ -203,8 +202,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         write_to_kafka(event, data.to_java_bytes)
       end
     else
-      raise ConfigurationError, "'value_serializer' only supports org.apache.kafka.common.serialization.ByteArraySerializer and org.apache.kafka.common.serialization.StringSerializer" 
+      raise LogStash::ConfigurationError, "'value_serializer' only supports org.apache.kafka.common.serialization.ByteArraySerializer and org.apache.kafka.common.serialization.StringSerializer"
     end
+    @producer = create_producer
   end
 
   def prepare(record)

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.3.2'
+  s.version         = '11.3.4'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/integration/inputs/kafka_spec.rb

@@ -301,7 +301,7 @@ def consume_messages(config, queue: Queue.new, timeout:, event_count:)
 end
 
 
-describe "schema registry connection options" do
+describe "schema registry connection options", :integration => true do
   schema_registry = Manticore::Client.new
   before (:all) do
     shutdown_schema_registry

data/spec/unit/inputs/kafka_spec.rb

@@ -134,6 +134,21 @@ describe LogStash::Inputs::Kafka do
       it 'should set the consumer thread name' do
         expect(subject.instance_variable_get('@runner_threads').first.get_name).to eq("kafka-input-worker-test-0")
       end
+
+      context 'with records value frozen' do
+        # boolean, module name & nil .to_s are frozen by default (https://bugs.ruby-lang.org/issues/16150)
+        let(:payload) do [
+          org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "nil", nil),
+          org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "true", true),
+          org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "false", false),
+          org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "frozen", "".freeze)
+        ]
+        end
+
+        it "should process events" do
+          expect(q.size).to eq(4)
+        end
+      end
     end
 
     context 'when errors are encountered during poll' do

data/spec/unit/outputs/kafka_spec.rb

@@ -221,6 +221,26 @@ describe "outputs/kafka" do
         kafka.multi_receive([event])
       end
     end
+    context 'when retries is -1' do
+      let(:retries) { -1 }
+
+      it "should raise a Configuration error" do
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        expect { kafka.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
+  describe "value_serializer" do
+    let(:output) { LogStash::Plugin.lookup("output", "kafka").new(config) }
+
+    context "when a random string is set" do
+      let(:config) { { "topic_id" => "random", "value_serializer" => "test_string" } }
+
+      it "raises a ConfigurationError" do
+        expect { output.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
   end
 
   context 'when ssl endpoint identification disabled' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.3.2
+  version: 11.3.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-16 00:00:00.000000000 Z
+date: 2024-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement