ruby-kafka 0.7.4 → 1.1.0

data/lib/kafka/fetcher.rb

@@ -4,11 +4,11 @@ require "kafka/fetch_operation"
 
 module Kafka
   class Fetcher
-    attr_reader :queue
+    attr_reader :queue, :max_wait_time
 
     def initialize(cluster:, logger:, instrumenter:, max_queue_size:, group:)
       @cluster = cluster
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
       @max_queue_size = max_queue_size
       @group = group
@@ -17,6 +17,9 @@ module Kafka
       @commands = Queue.new
       @next_offsets = Hash.new { |h, k| h[k] = {} }
 
+      # We are only running when someone calls start.
+      @running = false
+
       # Long poll until at least this many bytes can be fetched.
       @min_bytes = 1
 
@@ -49,20 +52,21 @@ module Kafka
     def start
       return if @running
 
+      @running = true
+
       @thread = Thread.new do
         while @running
           loop
         end
-        @logger.info "Fetcher thread exited."
+        @logger.info "#{@group} Fetcher thread exited."
       end
       @thread.abort_on_exception = true
-
-      @running = true
     end
 
     def stop
       return unless @running
       @commands << [:stop, []]
+      @thread.join
     end
 
     def reset
@@ -93,6 +97,7 @@ module Kafka
     attr_reader :current_reset_counter
 
     def loop
+      @logger.push_tags(@group.to_s)
       @instrumenter.instrument("loop.fetcher", {
         queue_size: @queue.size,
       })
@@ -108,9 +113,11 @@ module Kafka
       elsif @queue.size < @max_queue_size
         step
       else
-        @logger.warn "Reached max fetcher queue size (#{@max_queue_size}), sleeping 1s"
+        @logger.info "Reached max fetcher queue size (#{@max_queue_size}), sleeping 1s"
         sleep 1
       end
+    ensure
+      @logger.pop_tags
     end
 
     def handle_configure(min_bytes, max_bytes, max_wait_time)

data/lib/kafka/gzip_codec.rb

@@ -6,6 +6,10 @@ module Kafka
       1
     end
 
+    def produce_api_min_version
+      0
+    end
+
     def load
       require "zlib"
     end

data/lib/kafka/heartbeat.rb

@@ -2,15 +2,20 @@
 
 module Kafka
   class Heartbeat
-    def initialize(group:, interval:)
+    def initialize(group:, interval:, instrumenter:)
       @group = group
       @interval = interval
       @last_heartbeat = Time.now
+      @instrumenter = instrumenter
     end
 
     def trigger!
-      @group.heartbeat
-      @last_heartbeat = Time.now
+      @instrumenter.instrument('heartbeat.consumer',
+                               group_id: @group.group_id,
+                               topic_partitions: @group.assigned_partitions) do
+        @group.heartbeat
+        @last_heartbeat = Time.now
+      end
     end
 
     def trigger

data/lib/kafka/lz4_codec.rb

@@ -6,6 +6,10 @@ module Kafka
       3
     end
 
+    def produce_api_min_version
+      0
+    end
+
     def load
       require "extlz4"
     rescue LoadError

data/lib/kafka/offset_manager.rb

@@ -13,7 +13,7 @@ module Kafka
       @cluster = cluster
       @group = group
       @fetcher = fetcher
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @commit_interval = commit_interval
       @commit_threshold = commit_threshold
 
@@ -50,9 +50,20 @@ module Kafka
     # @param offset [Integer] the offset of the message that should be marked as processed.
     # @return [nil]
     def mark_as_processed(topic, partition, offset)
-      @uncommitted_offsets += 1
+      unless @group.assigned_to?(topic, partition)
+        @logger.debug "Not marking #{topic}/#{partition}:#{offset} as processed for partition not assigned to this consumer."
+        return
+      end
       @processed_offsets[topic] ||= {}
 
+      last_processed_offset = @processed_offsets[topic][partition] || -1
+      if last_processed_offset > offset + 1
+        @logger.debug "Not overwriting newer offset #{topic}/#{partition}:#{last_processed_offset - 1} with older #{offset}"
+        return
+      end
+
+      @uncommitted_offsets += 1
+
       # The committed offset should always be the offset of the next message that the
       # application will read, thus adding one to the last message processed.
       @processed_offsets[topic][partition] = offset + 1

data/lib/kafka/produce_operation.rb

@@ -37,7 +37,7 @@ module Kafka
       @required_acks = required_acks
       @ack_timeout = ack_timeout
       @compressor = compressor
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
     end
 

data/lib/kafka/producer.rb

@@ -68,6 +68,8 @@ module Kafka
   #
   # * `:snappy` for [Snappy](http://google.github.io/snappy/) compression.
   # * `:gzip` for [gzip](https://en.wikipedia.org/wiki/Gzip) compression.
+  # * `:lz4` for [LZ4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) compression.
+  # * `:zstd` for [zstd](https://facebook.github.io/zstd/) compression.
   #
   # By default, all message sets will be compressed if you specify a compression
   # codec. To increase the compression threshold, set `compression_threshold` to
@@ -130,7 +132,7 @@ module Kafka
     def initialize(cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
       @cluster = cluster
       @transaction_manager = transaction_manager
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
       @required_acks = required_acks == :all ? -1 : required_acks
       @ack_timeout = ack_timeout
@@ -150,6 +152,10 @@ module Kafka
       @pending_message_queue = PendingMessageQueue.new
     end
 
+    def to_s
+      "Producer #{@target_topics.to_a.join(', ')}"
+    end
+
     # Produces a message to the specified topic. Note that messages are buffered in
     # the producer until {#deliver_messages} is called.
     #
@@ -182,11 +188,14 @@ module Kafka
     # @raise [BufferOverflow] if the maximum buffer size has been reached.
     # @return [nil]
     def produce(value, key: nil, headers: {}, topic:, partition: nil, partition_key: nil, create_time: Time.now)
+      # We want to fail fast if `topic` isn't a String
+      topic = topic.to_str
+
       message = PendingMessage.new(
         value: value && value.to_s,
         key: key && key.to_s,
         headers: headers,
-        topic: topic.to_s,
+        topic: topic,
         partition: partition && Integer(partition),
         partition_key: partition_key && partition_key.to_s,
         create_time: create_time
@@ -205,7 +214,7 @@ module Kafka
       # If the producer is in transactional mode, all the message production
       # must be used when the producer is currently in transaction
       if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
-        raise 'You must trigger begin_transaction before producing messages'
+        raise "Cannot produce to #{topic}: You must trigger begin_transaction before producing messages"
       end
 
       @target_topics.add(topic)
@@ -324,6 +333,20 @@ module Kafka
       @transaction_manager.abort_transaction
     end
 
+    # Sends batch last offset to the consumer group coordinator, and also marks
+    # this offset as part of the current transaction. This offset will be considered
+    # committed only if the transaction is committed successfully.
+    #
+    # This method should be used when you need to batch consumed and produced messages
+    # together, typically in a consume-transform-produce pattern. Thus, the specified
+    # group_id should be the same as config parameter group_id of the used
+    # consumer.
+    #
+    # @return [nil]
+    def send_offsets_to_transaction(batch:, group_id:)
+      @transaction_manager.send_offsets_to_txn(offsets: { batch.topic => { batch.partition => { offset: batch.last_offset + 1, leader_epoch: batch.leader_epoch } } }, group_id: group_id)
+    end
+
     # Syntactic sugar to enable easier transaction usage. Do the following steps
     #
     # - Start the transaction (with Producer#begin_transaction)
@@ -391,11 +414,11 @@ module Kafka
         if buffer_size.zero?
           break
         elsif attempt <= @max_retries
-          @logger.warn "Failed to send all messages; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
+          @logger.warn "Failed to send all messages to #{pretty_partitions}; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
 
           sleep @retry_backoff
         else
-          @logger.error "Failed to send all messages; keeping remaining messages in buffer"
+          @logger.error "Failed to send all messages to #{pretty_partitions}; keeping remaining messages in buffer"
           break
         end
       end
@@ -407,12 +430,14 @@ module Kafka
       end
 
       unless @buffer.empty?
-        partitions = @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
-
-        raise DeliveryFailed.new("Failed to send messages to #{partitions}", buffer_messages)
+        raise DeliveryFailed.new("Failed to send messages to #{pretty_partitions}", buffer_messages)
       end
     end
 
+    def pretty_partitions
+      @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
+    end
+
     def assign_partitions!
       failed_messages = []
       topics_with_failures = Set.new

data/lib/kafka/prometheus.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+#
+#  Subscriber to ruby_kafka to report metrics to prometheus
+#
+#  Usage:
+#     require "kafka/prometheus"
+#
+#  Once the file has been required, no further configuration is needed, all operational
+#  metrics are automatically emitted (Unless PROMETHEUS_NO_AUTO_START is set).
+#
+#  By Peter Mustel, T2 Data AB
+#
+begin
+  require 'prometheus/client'
+rescue LoadError
+  warn 'In order to report Kafka client metrics to Prometheus you need to install the `prometheus-client` gem.'
+  raise
+end
+
+require 'active_support/subscriber'
+
+module Kafka
+  module Prometheus
+    SIZE_BUCKETS = [1, 10, 100, 1000, 10_000, 100_000, 1_000_000].freeze
+    LATENCY_BUCKETS = [0.0001, 0.001, 0.01, 0.1, 1.0, 10, 100, 1000].freeze
+    DELAY_BUCKETS = [1, 3, 10, 30, 100, 300, 1000, 3000, 10_000, 30_000].freeze
+
+    class << self
+      attr_accessor :registry
+
+      def start(registry = ::Prometheus::Client.registry)
+        @registry = registry
+        ConnectionSubscriber.attach_to 'connection.kafka'
+        ConsumerSubscriber.attach_to 'consumer.kafka'
+        ProducerSubscriber.attach_to 'producer.kafka'
+        AsyncProducerSubscriber.attach_to 'async_producer.kafka'
+        FetcherSubscriber.attach_to 'fetcher.kafka'
+      end
+    end
+
+    class ConnectionSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @api_calls = Prometheus.registry.counter(:api_calls, docstring: 'Total calls', labels: [:client, :api, :broker])
+        @api_latency = Prometheus.registry.histogram(:api_latency, docstring: 'Latency', buckets: LATENCY_BUCKETS, labels: [:client, :api, :broker])
+        @api_request_size = Prometheus.registry.histogram(:api_request_size, docstring: 'Request size', buckets: SIZE_BUCKETS, labels: [:client, :api, :broker])
+        @api_response_size = Prometheus.registry.histogram(:api_response_size, docstring: 'Response size', buckets: SIZE_BUCKETS, labels: [:client, :api, :broker])
+        @api_errors = Prometheus.registry.counter(:api_errors, docstring: 'Errors', labels: [:client, :api, :broker])
+      end
+
+      def request(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          api: event.payload.fetch(:api, 'unknown'),
+          broker: event.payload.fetch(:broker_host)
+        }
+        request_size = event.payload.fetch(:request_size, 0)
+        response_size = event.payload.fetch(:response_size, 0)
+
+        @api_calls.increment(labels: key)
+        @api_latency.observe(event.duration, labels: key)
+        @api_request_size.observe(request_size, labels: key)
+        @api_response_size.observe(response_size, labels: key)
+        @api_errors.increment(labels: key) if event.payload.key?(:exception)
+      end
+    end
+
+    class ConsumerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @process_messages = Prometheus.registry.counter(:consumer_process_messages, docstring: 'Total messages', labels: [:client, :group_id, :topic, :partition])
+        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, docstring: 'Total errors', labels: [:client, :group_id, :topic, :partition])
+        @process_message_latency =
+          Prometheus.registry.histogram(:consumer_process_message_latency, docstring: 'Latency', buckets: LATENCY_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, docstring: 'Offset lag', labels: [:client, :group_id, :topic, :partition])
+        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, docstring: 'Time lag of message', labels: [:client, :group_id, :topic, :partition])
+        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, docstring: 'Total errors in batch', labels: [:client, :group_id, :topic, :partition])
+        @process_batch_latency =
+          Prometheus.registry.histogram(:consumer_process_batch_latency, docstring: 'Latency in batch', buckets: LATENCY_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, docstring: 'Size of batch', buckets: SIZE_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @join_group = Prometheus.registry.histogram(:consumer_join_group, docstring: 'Time to join group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, docstring: 'Total error in joining group', labels: [:client, :group_id])
+        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, docstring: 'Time to sync group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, docstring: 'Total error in syncing group', labels: [:client, :group_id])
+        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, docstring: 'Time to leave group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, docstring: 'Total error in leaving group', labels: [:client, :group_id])
+        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, docstring: 'Pause duration', labels: [:client, :group_id, :topic, :partition])
+      end
+
+      def process_message(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+
+        offset_lag = event.payload.fetch(:offset_lag)
+        create_time = event.payload.fetch(:create_time)
+
+        time_lag = create_time && ((Time.now - create_time) * 1000).to_i
+
+        if event.payload.key?(:exception)
+          @process_message_errors.increment(labels: key)
+        else
+          @process_message_latency.observe(event.duration, labels: key)
+          @process_messages.increment(labels: key)
+        end
+
+        @offset_lag.set(offset_lag, labels: key)
+
+        # Not all messages have timestamps.
+        return unless time_lag
+
+        @time_lag.set(time_lag, labels: key)
+      end
+
+      def process_batch(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+        message_count = event.payload.fetch(:message_count)
+
+        if event.payload.key?(:exception)
+          @process_batch_errors.increment(labels: key)
+        else
+          @process_batch_latency.observe(event.duration, labels: key)
+          @process_messages.increment(by: message_count, labels: key)
+        end
+      end
+
+      def fetch_batch(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+        offset_lag = event.payload.fetch(:offset_lag)
+        batch_size = event.payload.fetch(:message_count)
+
+        @batch_size.observe(batch_size, labels: key)
+        @offset_lag.set(offset_lag, labels: key)
+      end
+
+      def join_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @join_group.observe(event.duration, labels: key)
+
+        @join_group_errors.increment(labels: key) if event.payload.key?(:exception)
+      end
+
+      def sync_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @sync_group.observe(event.duration, labels: key)
+
+        @sync_group_errors.increment(labels: key) if event.payload.key?(:exception)
+      end
+
+      def leave_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @leave_group.observe(event.duration, labels: key)
+
+        @leave_group_errors.increment(labels: key) if event.payload.key?(:exception)
+      end
+
+      def pause_status(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+
+        duration = event.payload.fetch(:duration)
+        @pause_duration.set(duration, labels: key)
+      end
+    end
+
+    class ProducerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, docstring: 'Produced messages total', labels: [:client, :topic])
+        @produce_message_size =
+          Prometheus.registry.histogram(:producer_message_size, docstring: 'Message size', buckets: SIZE_BUCKETS, labels: [:client, :topic])
+        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, docstring: 'Buffer size', buckets: SIZE_BUCKETS, labels: [:client])
+        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, docstring: 'Buffer fill ratio', labels: [:client])
+        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, docstring: 'Buffer fill percentage', labels: [:client])
+        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, docstring: 'Produce errors', labels: [:client, :topic])
+        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, docstring: 'Deliver error', labels: [:client])
+        @deliver_latency =
+          Prometheus.registry.histogram(:producer_deliver_latency, docstring: 'Delivery latency', buckets: LATENCY_BUCKETS, labels: [:client])
+        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, docstring: 'Total count of delivered messages', labels: [:client])
+        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, docstring: 'Delivery attempts', labels: [:client])
+        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, docstring: 'Ack', labels: [:client, :topic])
+        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, docstring: 'Ack delay', buckets: LATENCY_BUCKETS, labels: [:client, :topic])
+        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, docstring: 'Ack errors', labels: [:client, :topic])
+      end
+
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        key = { client: client, topic: event.payload.fetch(:topic) }
+
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+        max_buffer_size = event.payload.fetch(:max_buffer_size)
+        buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
+        buffer_fill_percentage = buffer_fill_ratio * 100.0
+
+        # This gets us the write rate.
+        @produce_messages.increment(labels: key)
+        @produce_message_size.observe(message_size, labels: key)
+
+        # This gets us the avg/max buffer size per producer.
+        @buffer_size.observe(buffer_size, labels: { client: client })
+
+        # This gets us the avg/max buffer fill ratio per producer.
+        @buffer_fill_ratio.observe(buffer_fill_ratio, labels: { client: client })
+        @buffer_fill_percentage.observe(buffer_fill_percentage, labels: { client: client })
+      end
+
+      def buffer_overflow(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+        @produce_errors.increment(labels: key)
+      end
+
+      def deliver_messages(event)
+        key = { client: event.payload.fetch(:client_id) }
+        message_count = event.payload.fetch(:delivered_message_count)
+        attempts = event.payload.fetch(:attempts)
+
+        @deliver_errors.increment(labels: key) if event.payload.key?(:exception)
+        @deliver_latency.observe(event.duration, labels: key)
+
+        # Messages delivered to Kafka:
+        @deliver_messages.increment(by: message_count, labels: key)
+
+        # Number of attempts to deliver messages:
+        @deliver_attempts.observe(attempts, labels: key)
+      end
+
+      def ack_message(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        # Number of messages ACK'd for the topic.
+        @ack_messages.increment(labels: key)
+
+        # Histogram of delay between a message being produced and it being ACK'd.
+        @ack_delay.observe(event.payload.fetch(:delay), labels: key)
+      end
+
+      def topic_error(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        @ack_errors.increment(labels: key)
+      end
+    end
+
+    class AsyncProducerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, docstring: 'Queue size', buckets: SIZE_BUCKETS, labels: [:client, :topic])
+        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, docstring: 'Queue fill ratio', labels: [:client, :topic])
+        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, docstring: 'Producer errors', labels: [:client, :topic])
+        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, docstring: 'Dropped messages', labels: [:client])
+      end
+
+      def enqueue_message(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        queue_size = event.payload.fetch(:queue_size)
+        max_queue_size = event.payload.fetch(:max_queue_size)
+        queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
+
+        # This gets us the avg/max queue size per producer.
+        @queue_size.observe(queue_size, labels: key)
+
+        # This gets us the avg/max queue fill ratio per producer.
+        @queue_fill_ratio.observe(queue_fill_ratio, labels: key)
+      end
+
+      def buffer_overflow(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+        @produce_errors.increment(labels: key)
+      end
+
+      def drop_messages(event)
+        key = { client: event.payload.fetch(:client_id) }
+        message_count = event.payload.fetch(:message_count)
+        @dropped_messages.increment(by: message_count, labels: key)
+      end
+    end
+
+    class FetcherSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, docstring: 'Queue size', labels: [:client, :group_id])
+      end
+
+      def loop(event)
+        queue_size = event.payload.fetch(:queue_size)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        @queue_size.set(queue_size, labels: { client: client, group_id: group_id })
+      end
+    end
+  end
+end
+
+# To enable testability, it is possible to skip the start until test time
+Kafka::Prometheus.start unless defined?(PROMETHEUS_NO_AUTO_START)

data/lib/kafka/protocol/add_offsets_to_txn_request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddOffsetsToTxnRequest
+      def initialize(transactional_id: nil, producer_id:, producer_epoch:, group_id:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @group_id = group_id
+      end
+
+      def api_key
+        ADD_OFFSETS_TO_TXN_API
+      end
+
+      def response_class
+        AddOffsetsToTxnResponse
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_string(@group_id)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/add_offsets_to_txn_response.rb

@@ -0,0 +1,19 @@
+module Kafka
+  module Protocol
+    class AddOffsetsToTxnResponse
+
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+
+    end
+  end
+end

data/lib/kafka/protocol/join_group_request.rb

@@ -7,13 +7,14 @@ module Kafka
     class JoinGroupRequest
       PROTOCOL_TYPE = "consumer"
 
-      def initialize(group_id:, session_timeout:, member_id:, topics: [])
+      def initialize(group_id:, session_timeout:, rebalance_timeout:, member_id:, topics: [])
         @group_id = group_id
         @session_timeout = session_timeout * 1000 # Kafka wants ms.
+        @rebalance_timeout = rebalance_timeout * 1000 # Kafka wants ms.
         @member_id = member_id || ""
         @protocol_type = PROTOCOL_TYPE
         @group_protocols = {
-          "standard" => ConsumerGroupProtocol.new(topics: ["test-messages"]),
+          "roundrobin" => ConsumerGroupProtocol.new(topics: topics),
         }
       end
 
@@ -21,6 +22,10 @@ module Kafka
         JOIN_GROUP_API
       end
 
+      def api_version
+        1
+      end
+
       def response_class
         JoinGroupResponse
       end
@@ -28,6 +33,7 @@ module Kafka
       def encode(encoder)
         encoder.write_string(@group_id)
         encoder.write_int32(@session_timeout)
+        encoder.write_int32(@rebalance_timeout)
         encoder.write_string(@member_id)
         encoder.write_string(@protocol_type)
 

data/lib/kafka/protocol/metadata_response.rb

@@ -34,7 +34,7 @@ module Kafka
     #
     class MetadataResponse
       class PartitionMetadata
-        attr_reader :partition_id, :leader
+        attr_reader :partition_id, :leader, :replicas
 
         attr_reader :partition_error_code