logstash-integration-kafka 10.0.1-java → 10.5.0-java

data/lib/logstash/outputs/kafka.rb

@@ -67,7 +67,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The producer will attempt to batch records together into fewer requests whenever multiple
   # records are being sent to the same partition. This helps performance on both the client
   # and the server. This configuration controls the default batch size in bytes.
-  config :batch_size, :validate => :number, :default => 16384
+  config :batch_size, :validate => :number, :default => 16_384 # Kafka default
   # This is for bootstrapping and the producer will only use it for getting metadata (topics,
   # partitions and replicas). The socket connections for sending the actual data will be
   # established based on the broker information returned in the metadata. The format is
@@ -75,10 +75,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # subset of brokers.
   config :bootstrap_servers, :validate => :string, :default => 'localhost:9092'
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
-  config :buffer_memory, :validate => :number, :default => 33554432
+  config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
   # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
   config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
+  # IP addresses for a hostname, they will all be attempted to connect to before failing the 
+  # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+  # resolved and expanded into a list of canonical names.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
   # The id string to pass to the server when making requests.
   # The purpose of this is to be able to track the source of requests beyond just
   # ip/port by allowing a logical application name to be included with the request
@@ -92,24 +97,26 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # This setting accomplishes this by adding a small amount of artificial delay—that is,
   # rather than immediately sending out a record the producer will wait for up to the given delay
   # to allow other records to be sent so that the sends can be batched together.
-  config :linger_ms, :validate => :number, :default => 0
+  config :linger_ms, :validate => :number, :default => 0 # Kafka default
   # The maximum size of a request
-  config :max_request_size, :validate => :number, :default => 1048576
+  config :max_request_size, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The key for the message
   config :message_key, :validate => :string
   # the timeout setting for initial metadata request to fetch topic metadata.
-  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60000
+  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
   # the max time in milliseconds before a metadata refresh is forced.
-  config :metadata_max_age_ms, :validate => :number, :default => 300000
+  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
+  # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
+  config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
-  config :receive_buffer_bytes, :validate => :number, :default => 32768
+  config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
-  config :reconnect_backoff_ms, :validate => :number, :default => 10
+  config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
   # The configuration controls the maximum amount of time the client will wait
   # for the response of a request. If the response is not received before the timeout
   # elapses the client will resend the request if necessary or fail the request if
   # retries are exhausted.
-  config :request_timeout_ms, :validate => :string
+  config :request_timeout_ms, :validate => :number, :default => 40_000 # (40s) Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #
@@ -120,9 +127,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # A value less than zero is a configuration error.
   config :retries, :validate => :number
   # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
-  config :retry_backoff_ms, :validate => :number, :default => 100
+  config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
   # The size of the TCP send buffer to use when sending data.
-  config :send_buffer_bytes, :validate => :number, :default => 131072
+  config :send_buffer_bytes, :validate => :number, :default => 131_072 # (128KB) Kafka default
   # The truststore type.
   config :ssl_truststore_type, :validate => :string
   # The JKS truststore path to validate the Kafka broker's certificate.
@@ -184,7 +191,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
       end
 
-      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
+      logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
     end
 
 
@@ -202,8 +209,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
   end
 
-  # def register
-
   def prepare(record)
     # This output is threadsafe, so we need to keep a batch per thread.
     @thread_batch_map[Thread.current].add(record)
@@ -231,7 +236,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     remaining = @retries
 
     while batch.any?
-      if !remaining.nil?
+      unless remaining.nil?
         if remaining < 0
           # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
           # DLQing would make things worse (you dlq data that would be successful
@@ -250,27 +255,39 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         begin
           # send() can throw an exception even before the future is created.
           @producer.send(record)
-        rescue org.apache.kafka.common.errors.TimeoutException => e
-          failures << record
-          nil
-        rescue org.apache.kafka.common.errors.InterruptException => e
+        rescue org.apache.kafka.common.errors.InterruptException,
+               org.apache.kafka.common.errors.RetriableException => e
+          logger.info("producer send failed, will retry sending", :exception => e.class, :message => e.message)
           failures << record
           nil
-        rescue org.apache.kafka.common.errors.SerializationException => e
-          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
-          # TODO(sissel): Let's add DLQ here.
-          failures << record
+        rescue org.apache.kafka.common.KafkaException => e
+          # This error is not retriable, drop event
+          # TODO: add DLQ support
+          logger.warn("producer send failed, dropping record",:exception => e.class, :message => e.message,
+                      :record_value => record.value)
           nil
         end
-      end.compact
+      end
 
       futures.each_with_index do |future, i|
-        begin
-          result = future.get()
-        rescue => e
-          # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("KafkaProducer.send() failed: #{e}", :exception => e)
-          failures << batch[i]
+        # We cannot skip nils using `futures.compact` because then our index `i` will not align with `batch`
+        unless future.nil?
+          begin
+            future.get
+          rescue java.util.concurrent.ExecutionException => e
+            # TODO(sissel): Add metric to count failures, possibly by exception type.
+            if e.get_cause.is_a? org.apache.kafka.common.errors.RetriableException or
+               e.get_cause.is_a? org.apache.kafka.common.errors.InterruptException
+              logger.info("producer send failed, will retry sending", :exception => e.cause.class,
+                          :message => e.cause.message)
+              failures << batch[i]
+            elsif e.get_cause.is_a? org.apache.kafka.common.KafkaException
+              # This error is not retriable, drop event
+              # TODO: add DLQ support
+              logger.warn("producer send failed, dropping record", :exception => e.cause.class,
+                          :message => e.cause.message, :record_value => batch[i].value)
+            end
+          end
         end
       end
 
@@ -303,10 +320,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
     prepare(record)
   rescue LogStash::ShutdownSignal
-    @logger.debug('Kafka producer got shutdown signal')
+    logger.debug('producer received shutdown signal')
   rescue => e
-    @logger.warn('kafka producer threw exception, restarting',
-                 :exception => e)
+    logger.warn('producer threw exception, restarting', :exception => e.class, :message => e.message)
   end
 
   def create_producer
@@ -319,14 +335,19 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers)
       props.put(kafka::BUFFER_MEMORY_CONFIG, buffer_memory.to_s)
       props.put(kafka::COMPRESSION_TYPE_CONFIG, compression_type)
+      props.put(kafka::CLIENT_DNS_LOOKUP_CONFIG, client_dns_lookup)
       props.put(kafka::CLIENT_ID_CONFIG, client_id) unless client_id.nil?
       props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, key_serializer)
       props.put(kafka::LINGER_MS_CONFIG, linger_ms.to_s)
       props.put(kafka::MAX_REQUEST_SIZE_CONFIG, max_request_size.to_s)
-      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
+      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms.to_s) unless metadata_max_age_ms.nil?
+      unless partitioner.nil?
+        props.put(kafka::PARTITIONER_CLASS_CONFIG, partitioner = partitioner_class)
+        logger.debug('producer configured using partitioner', :partitioner_class => partitioner)
+      end
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
-      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
-      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s)
@@ -343,7 +364,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         set_sasl_config(props)
       end
 
-
       org.apache.kafka.clients.producer.KafkaProducer.new(props)
     rescue => e
       logger.error("Unable to create Kafka producer from given configuration",
@@ -353,13 +373,31 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
   end
 
+  def partitioner_class
+    case partitioner
+    when 'round_robin'
+      'org.apache.kafka.clients.producer.RoundRobinPartitioner'
+    when 'uniform_sticky'
+      'org.apache.kafka.clients.producer.UniformStickyPartitioner'
+    when 'default'
+      'org.apache.kafka.clients.producer.internals.DefaultPartitioner'
+    else
+      unless partitioner.index('.')
+        raise LogStash::ConfigurationError, "unsupported partitioner: #{partitioner.inspect}"
+      end
+      partitioner # assume a fully qualified class-name
+    end
+  end
+
   def set_trustore_keystore_config(props)
-    if ssl_truststore_location.nil?
-      raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
+    unless ssl_endpoint_identification_algorithm.to_s.strip.empty?
+      if ssl_truststore_location.nil?
+        raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
+      end
+      props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+      props.put("ssl.truststore.location", ssl_truststore_location)
+      props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
     end
-    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
-    props.put("ssl.truststore.location", ssl_truststore_location)
-    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
 
     # Client auth stuff
     props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
@@ -370,15 +408,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   end
 
   def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config",jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf",kerberos_config) unless kerberos_config.nil?
+    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
 
     props.put("sasl.mechanism",sasl_mechanism)
     if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
       raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
     end
 
-    props.put("sasl.kerberos.service.name",sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
     props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
   end
 

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.0.1'
+  s.version         = '10.5.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -49,6 +49,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'poseidon'
+  s.add_development_dependency 'ruby-kafka'
   s.add_development_dependency 'snappy'
 end

data/spec/integration/inputs/kafka_spec.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
-require "digest"
 require "rspec/wait"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
@@ -12,159 +11,129 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_3) {rand(36**8).to_s(36)}
   let(:group_id_4) {rand(36**8).to_s(36)}
   let(:group_id_5) {rand(36**8).to_s(36)}
-  let(:plain_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:multi_consumer_config) { plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3}) }
-  let(:snappy_config) { { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:pattern_config) { { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:decorate_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
-  let(:manual_commit_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5, 'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false'} }
+  let(:group_id_6) {rand(36**8).to_s(36)}
+  let(:plain_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:multi_consumer_config) do
+    plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3})
+  end
+  let(:snappy_config) do
+    { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:lz4_config) do
+    { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:pattern_config) do
+    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain',
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:decorate_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+  end
+  let(:manual_commit_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
+      'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
+  end
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
   describe "#kafka-topics" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
 
     it "should consume all messages from plain 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(plain_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consume all messages from snappy 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(snappy_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consume all messages from lz4 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(lz4_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consumer all messages with multiple consumers" do
-      kafka_input = LogStash::Inputs::Kafka.new(multi_consumer_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+      consume_messages(multi_consumer_config, timeout: timeout_seconds, event_count: num_events) do |queue, kafka_input|
         expect(queue.length).to eq(num_events)
         kafka_input.kafka_consumers.each_with_index do |consumer, i|
           expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
         end
-      ensure
-        t.kill
-        t.join(30_000)
       end
     end
   end
 
-  describe "#kafka-topics-pattern" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
+  context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(3*num_events)
-        expect(queue.length).to eq(3*num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      total_events = num_events * 3
+      queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
+      expect(queue.length).to eq(total_events)
     end
   end
 
-  describe "#kafka-decorate" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
+  context "#kafka-decorate" do
     it "should show the right topic and group name in decorated kafka section" do
       start = LogStash::Timestamp.now.time.to_i
-      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+      consume_messages(decorate_config, timeout: timeout_seconds, event_count: num_events) do |queue, _|
         expect(queue.length).to eq(num_events)
         event = queue.shift
         expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain")
         expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
-      ensure
-        t.kill
-        t.join(30_000)
       end
     end
   end
 
-  describe "#kafka-offset-commit" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
+  context "#kafka-offset-commit" do
+    it "should manually commit offsets" do
+      queue = consume_messages(manual_commit_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
+    end
+  end
+
+  context 'setting partition_assignment_strategy' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:consumer_config) do
+      plain_config.merge(
+          "topics" => [test_topic],
+          'group_id' => group_id_6,
+          "client_id" => "partition_assignment_strategy-spec",
+          "consumer_threads" => 2,
+          "partition_assignment_strategy" => partition_assignment_strategy
+      )
+    end
+    let(:partition_assignment_strategy) { nil }
+
+    # NOTE: just verify setting works, as its a bit cumbersome to do in a unit spec
+    [ 'range', 'round_robin', 'sticky', 'org.apache.kafka.clients.consumer.CooperativeStickyAssignor' ].each do |partition_assignment_strategy|
+      describe partition_assignment_strategy do
+        let(:partition_assignment_strategy) { partition_assignment_strategy }
+        it 'consumes data' do
+          consume_messages(consumer_config, timeout: false, event_count: 0)
         end
       end
     end
+  end
 
-    it "should manually commit offsets" do
-      kafka_input = LogStash::Inputs::Kafka.new(manual_commit_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+  private
+
+  def consume_messages(config, queue: Queue.new, timeout:, event_count:)
+    kafka_input = LogStash::Inputs::Kafka.new(config)
+    t = Thread.new { kafka_input.run(queue) }
+    begin
+      t.run
+      wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
+      block_given? ? yield(queue, kafka_input) : queue
+    ensure
+      t.kill
+      t.join(30_000)
     end
   end
+
 end