ruby-kafka 0.7.10 → 1.5.0

data/lib/kafka/prometheus.rb

@@ -42,11 +42,11 @@ module Kafka
     class ConnectionSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @api_calls = Prometheus.registry.counter(:api_calls, 'Total calls')
-        @api_latency = Prometheus.registry.histogram(:api_latency, 'Latency', {}, LATENCY_BUCKETS)
-        @api_request_size = Prometheus.registry.histogram(:api_request_size, 'Request size', {}, SIZE_BUCKETS)
-        @api_response_size = Prometheus.registry.histogram(:api_response_size, 'Response size', {}, SIZE_BUCKETS)
-        @api_errors = Prometheus.registry.counter(:api_errors, 'Errors')
+        @api_calls = Prometheus.registry.counter(:api_calls, docstring: 'Total calls', labels: [:client, :api, :broker])
+        @api_latency = Prometheus.registry.histogram(:api_latency, docstring: 'Latency', buckets: LATENCY_BUCKETS, labels: [:client, :api, :broker])
+        @api_request_size = Prometheus.registry.histogram(:api_request_size, docstring: 'Request size', buckets: SIZE_BUCKETS, labels: [:client, :api, :broker])
+        @api_response_size = Prometheus.registry.histogram(:api_response_size, docstring: 'Response size', buckets: SIZE_BUCKETS, labels: [:client, :api, :broker])
+        @api_errors = Prometheus.registry.counter(:api_errors, docstring: 'Errors', labels: [:client, :api, :broker])
       end
 
       def request(event)
@@ -58,34 +58,34 @@ module Kafka
         request_size = event.payload.fetch(:request_size, 0)
         response_size = event.payload.fetch(:response_size, 0)
 
-        @api_calls.increment(key)
-        @api_latency.observe(key, event.duration)
-        @api_request_size.observe(key, request_size)
-        @api_response_size.observe(key, response_size)
-        @api_errors.increment(key) if event.payload.key?(:exception)
+        @api_calls.increment(labels: key)
+        @api_latency.observe(event.duration, labels: key)
+        @api_request_size.observe(request_size, labels: key)
+        @api_response_size.observe(response_size, labels: key)
+        @api_errors.increment(labels: key) if event.payload.key?(:exception)
       end
     end
 
     class ConsumerSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @process_messages = Prometheus.registry.counter(:consumer_process_messages, 'Total messages')
-        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, 'Total errors')
+        @process_messages = Prometheus.registry.counter(:consumer_process_messages, docstring: 'Total messages', labels: [:client, :group_id, :topic, :partition])
+        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, docstring: 'Total errors', labels: [:client, :group_id, :topic, :partition])
         @process_message_latency =
-          Prometheus.registry.histogram(:consumer_process_message_latency, 'Latency', {}, LATENCY_BUCKETS)
-        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, 'Offset lag')
-        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, 'Time lag of message')
-        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, 'Total errors in batch')
+          Prometheus.registry.histogram(:consumer_process_message_latency, docstring: 'Latency', buckets: LATENCY_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, docstring: 'Offset lag', labels: [:client, :group_id, :topic, :partition])
+        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, docstring: 'Time lag of message', labels: [:client, :group_id, :topic, :partition])
+        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, docstring: 'Total errors in batch', labels: [:client, :group_id, :topic, :partition])
         @process_batch_latency =
-          Prometheus.registry.histogram(:consumer_process_batch_latency, 'Latency in batch', {}, LATENCY_BUCKETS)
-        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, 'Size of batch', {}, SIZE_BUCKETS)
-        @join_group = Prometheus.registry.histogram(:consumer_join_group, 'Time to join group', {}, DELAY_BUCKETS)
-        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, 'Total error in joining group')
-        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, 'Time to sync group', {}, DELAY_BUCKETS)
-        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, 'Total error in syncing group')
-        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, 'Time to leave group', {}, DELAY_BUCKETS)
-        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, 'Total error in leaving group')
-        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, 'Pause duration')
+          Prometheus.registry.histogram(:consumer_process_batch_latency, docstring: 'Latency in batch', buckets: LATENCY_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, docstring: 'Size of batch', buckets: SIZE_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @join_group = Prometheus.registry.histogram(:consumer_join_group, docstring: 'Time to join group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, docstring: 'Total error in joining group', labels: [:client, :group_id])
+        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, docstring: 'Time to sync group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, docstring: 'Total error in syncing group', labels: [:client, :group_id])
+        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, docstring: 'Time to leave group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, docstring: 'Total error in leaving group', labels: [:client, :group_id])
+        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, docstring: 'Pause duration', labels: [:client, :group_id, :topic, :partition])
       end
 
       def process_message(event)
@@ -102,18 +102,18 @@ module Kafka
         time_lag = create_time && ((Time.now - create_time) * 1000).to_i
 
         if event.payload.key?(:exception)
-          @process_message_errors.increment(key)
+          @process_message_errors.increment(labels: key)
         else
-          @process_message_latency.observe(key, event.duration)
-          @process_messages.increment(key)
+          @process_message_latency.observe(event.duration, labels: key)
+          @process_messages.increment(labels: key)
         end
 
-        @offset_lag.set(key, offset_lag)
+        @offset_lag.set(offset_lag, labels: key)
 
         # Not all messages have timestamps.
         return unless time_lag
 
-        @time_lag.set(key, time_lag)
+        @time_lag.set(time_lag, labels: key)
       end
 
       def process_batch(event)
@@ -126,10 +126,10 @@ module Kafka
         message_count = event.payload.fetch(:message_count)
 
         if event.payload.key?(:exception)
-          @process_batch_errors.increment(key)
+          @process_batch_errors.increment(labels: key)
         else
-          @process_batch_latency.observe(key, event.duration)
-          @process_messages.increment(key, message_count)
+          @process_batch_latency.observe(event.duration, labels: key)
+          @process_messages.increment(by: message_count, labels: key)
         end
       end
 
@@ -143,29 +143,29 @@ module Kafka
         offset_lag = event.payload.fetch(:offset_lag)
         batch_size = event.payload.fetch(:message_count)
 
-        @batch_size.observe(key, batch_size)
-        @offset_lag.set(key, offset_lag)
+        @batch_size.observe(batch_size, labels: key)
+        @offset_lag.set(offset_lag, labels: key)
       end
 
       def join_group(event)
         key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
-        @join_group.observe(key, event.duration)
+        @join_group.observe(event.duration, labels: key)
 
-        @join_group_errors.increment(key) if event.payload.key?(:exception)
+        @join_group_errors.increment(labels: key) if event.payload.key?(:exception)
       end
 
       def sync_group(event)
         key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
-        @sync_group.observe(key, event.duration)
+        @sync_group.observe(event.duration, labels: key)
 
-        @sync_group_errors.increment(key) if event.payload.key?(:exception)
+        @sync_group_errors.increment(labels: key) if event.payload.key?(:exception)
       end
 
       def leave_group(event)
         key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
-        @leave_group.observe(key, event.duration)
+        @leave_group.observe(event.duration, labels: key)
 
-        @leave_group_errors.increment(key) if event.payload.key?(:exception)
+        @leave_group_errors.increment(labels: key) if event.payload.key?(:exception)
       end
 
       def pause_status(event)
@@ -177,28 +177,28 @@ module Kafka
         }
 
         duration = event.payload.fetch(:duration)
-        @pause_duration.set(key, duration)
+        @pause_duration.set(duration, labels: key)
       end
     end
 
     class ProducerSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, 'Produced messages total')
+        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, docstring: 'Produced messages total', labels: [:client, :topic])
         @produce_message_size =
-          Prometheus.registry.histogram(:producer_message_size, 'Message size', {}, SIZE_BUCKETS)
-        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, 'Buffer size', {}, SIZE_BUCKETS)
-        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, 'Buffer fill ratio')
-        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, 'Buffer fill percentage')
-        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, 'Produce errors')
-        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, 'Deliver error')
+          Prometheus.registry.histogram(:producer_message_size, docstring: 'Message size', buckets: SIZE_BUCKETS, labels: [:client, :topic])
+        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, docstring: 'Buffer size', buckets: SIZE_BUCKETS, labels: [:client])
+        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, docstring: 'Buffer fill ratio', labels: [:client])
+        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, docstring: 'Buffer fill percentage', labels: [:client])
+        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, docstring: 'Produce errors', labels: [:client, :topic])
+        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, docstring: 'Deliver error', labels: [:client])
         @deliver_latency =
-          Prometheus.registry.histogram(:producer_deliver_latency, 'Delivery latency', {}, LATENCY_BUCKETS)
-        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, 'Total count of delivered messages')
-        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, 'Delivery attempts')
-        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, 'Ack')
-        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, 'Ack delay', {}, LATENCY_BUCKETS)
-        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, 'Ack errors')
+          Prometheus.registry.histogram(:producer_deliver_latency, docstring: 'Delivery latency', buckets: LATENCY_BUCKETS, labels: [:client])
+        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, docstring: 'Total count of delivered messages', labels: [:client])
+        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, docstring: 'Delivery attempts', labels: [:client])
+        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, docstring: 'Ack', labels: [:client, :topic])
+        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, docstring: 'Ack delay', buckets: LATENCY_BUCKETS, labels: [:client, :topic])
+        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, docstring: 'Ack errors', labels: [:client, :topic])
       end
 
       def produce_message(event)
@@ -212,20 +212,20 @@ module Kafka
         buffer_fill_percentage = buffer_fill_ratio * 100.0
 
         # This gets us the write rate.
-        @produce_messages.increment(key)
-        @produce_message_size.observe(key, message_size)
+        @produce_messages.increment(labels: key)
+        @produce_message_size.observe(message_size, labels: key)
 
         # This gets us the avg/max buffer size per producer.
-        @buffer_size.observe({ client: client }, buffer_size)
+        @buffer_size.observe(buffer_size, labels: { client: client })
 
         # This gets us the avg/max buffer fill ratio per producer.
-        @buffer_fill_ratio.observe({ client: client }, buffer_fill_ratio)
-        @buffer_fill_percentage.observe({ client: client }, buffer_fill_percentage)
+        @buffer_fill_ratio.observe(buffer_fill_ratio, labels: { client: client })
+        @buffer_fill_percentage.observe(buffer_fill_percentage, labels: { client: client })
       end
 
       def buffer_overflow(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
-        @produce_errors.increment(key)
+        @produce_errors.increment(labels: key)
       end
 
       def deliver_messages(event)
@@ -233,40 +233,40 @@ module Kafka
         message_count = event.payload.fetch(:delivered_message_count)
         attempts = event.payload.fetch(:attempts)
 
-        @deliver_errors.increment(key) if event.payload.key?(:exception)
-        @deliver_latency.observe(key, event.duration)
+        @deliver_errors.increment(labels: key) if event.payload.key?(:exception)
+        @deliver_latency.observe(event.duration, labels: key)
 
         # Messages delivered to Kafka:
-        @deliver_messages.increment(key, message_count)
+        @deliver_messages.increment(by: message_count, labels: key)
 
         # Number of attempts to deliver messages:
-        @deliver_attempts.observe(key, attempts)
+        @deliver_attempts.observe(attempts, labels: key)
       end
 
       def ack_message(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
 
         # Number of messages ACK'd for the topic.
-        @ack_messages.increment(key)
+        @ack_messages.increment(labels: key)
 
         # Histogram of delay between a message being produced and it being ACK'd.
-        @ack_delay.observe(key, event.payload.fetch(:delay))
+        @ack_delay.observe(event.payload.fetch(:delay), labels: key)
       end
 
       def topic_error(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
 
-        @ack_errors.increment(key)
+        @ack_errors.increment(labels: key)
       end
     end
 
     class AsyncProducerSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, 'Queue size', {}, SIZE_BUCKETS)
-        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, 'Queue fill ratio')
-        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, 'Producer errors')
-        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, 'Dropped messages')
+        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, docstring: 'Queue size', buckets: SIZE_BUCKETS, labels: [:client, :topic])
+        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, docstring: 'Queue fill ratio', labels: [:client, :topic])
+        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, docstring: 'Producer errors', labels: [:client, :topic])
+        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, docstring: 'Dropped messages', labels: [:client])
       end
 
       def enqueue_message(event)
@@ -277,29 +277,28 @@ module Kafka
         queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
 
         # This gets us the avg/max queue size per producer.
-        @queue_size.observe(key, queue_size)
+        @queue_size.observe(queue_size, labels: key)
 
         # This gets us the avg/max queue fill ratio per producer.
-        @queue_fill_ratio.observe(key, queue_fill_ratio)
+        @queue_fill_ratio.observe(queue_fill_ratio, labels: key)
       end
 
       def buffer_overflow(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
-        @produce_errors.increment(key)
+        @produce_errors.increment(labels: key)
       end
 
       def drop_messages(event)
         key = { client: event.payload.fetch(:client_id) }
         message_count = event.payload.fetch(:message_count)
-
-        @dropped_messages.increment(key, message_count)
+        @dropped_messages.increment(by: message_count, labels: key)
       end
     end
 
     class FetcherSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, 'Queue size')
+        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, docstring: 'Queue size', labels: [:client, :group_id])
       end
 
       def loop(event)
@@ -307,7 +306,7 @@ module Kafka
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
 
-        @queue_size.set({ client: client, group_id: group_id }, queue_size)
+        @queue_size.set(queue_size, labels: { client: client, group_id: group_id })
       end
     end
   end

data/lib/kafka/protocol/add_offsets_to_txn_response.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kafka
   module Protocol
     class AddOffsetsToTxnResponse

data/lib/kafka/protocol/encoder.rb

@@ -126,7 +126,7 @@ module Kafka
       # Writes an integer under varints serializing to the IO object.
       # https://developers.google.com/protocol-buffers/docs/encoding#varints
       #
-      # @param string [Integer]
+      # @param int [Integer]
       # @return [nil]
       def write_varint(int)
         int = int << 1

data/lib/kafka/protocol/join_group_request.rb

@@ -7,13 +7,14 @@ module Kafka
     class JoinGroupRequest
       PROTOCOL_TYPE = "consumer"
 
-      def initialize(group_id:, session_timeout:, member_id:, topics: [])
+      def initialize(group_id:, session_timeout:, rebalance_timeout:, member_id:, topics: [], protocol_name:, user_data: nil)
         @group_id = group_id
         @session_timeout = session_timeout * 1000 # Kafka wants ms.
+        @rebalance_timeout = rebalance_timeout * 1000 # Kafka wants ms.
         @member_id = member_id || ""
         @protocol_type = PROTOCOL_TYPE
         @group_protocols = {
-          "standard" => ConsumerGroupProtocol.new(topics: ["test-messages"]),
+          protocol_name => ConsumerGroupProtocol.new(topics: topics, user_data: user_data),
         }
       end
 
@@ -21,6 +22,10 @@ module Kafka
         JOIN_GROUP_API
       end
 
+      def api_version
+        1
+      end
+
       def response_class
         JoinGroupResponse
       end
@@ -28,6 +33,7 @@ module Kafka
       def encode(encoder)
         encoder.write_string(@group_id)
         encoder.write_int32(@session_timeout)
+        encoder.write_int32(@rebalance_timeout)
         encoder.write_string(@member_id)
         encoder.write_string(@protocol_type)
 

data/lib/kafka/protocol/join_group_response.rb

@@ -3,6 +3,8 @@
 module Kafka
   module Protocol
     class JoinGroupResponse
+      Metadata = Struct.new(:version, :topics, :user_data)
+
       attr_reader :error_code
 
       attr_reader :generation_id, :group_protocol
@@ -25,7 +27,13 @@ module Kafka
           group_protocol: decoder.string,
           leader_id: decoder.string,
           member_id: decoder.string,
-          members: Hash[decoder.array { [decoder.string, decoder.bytes] }],
+          members: Hash[
+            decoder.array do
+              member_id = decoder.string
+              d = Decoder.from_string(decoder.bytes)
+              [member_id, Metadata.new(d.int16, d.array { d.string }, d.bytes)]
+            end
+          ],
         )
       end
     end

data/lib/kafka/protocol/metadata_response.rb

@@ -34,7 +34,7 @@ module Kafka
     #
     class MetadataResponse
       class PartitionMetadata
-        attr_reader :partition_id, :leader
+        attr_reader :partition_id, :leader, :replicas
 
         attr_reader :partition_error_code
 

data/lib/kafka/protocol/offset_fetch_request.rb

@@ -12,8 +12,10 @@ module Kafka
         OFFSET_FETCH_API
       end
 
+      # setting topics to nil fetches all offsets for a consumer group
+      # and that feature is only available in API version 2+
       def api_version
-        1
+        @topics.nil? ? 2 : 1
       end
 
       def response_class

data/lib/kafka/protocol/record_batch.rb

@@ -77,7 +77,7 @@ module Kafka
         record_batch_encoder.write_int8(MAGIC_BYTE)
 
         body = encode_record_batch_body
-        crc = Digest::CRC32c.checksum(body)
+        crc = ::Digest::CRC32c.checksum(body)
 
         record_batch_encoder.write_int32(crc)
         record_batch_encoder.write(body)
@@ -213,7 +213,7 @@ module Kafka
       end
 
       def mark_control_record
-        if in_transaction && is_control_batch
+        if is_control_batch
           record = @records.first
           record.is_control_record = true unless record.nil?
         end

data/lib/kafka/protocol/sasl_handshake_request.rb

@@ -8,7 +8,7 @@ module Kafka
 
     class SaslHandshakeRequest
 
-      SUPPORTED_MECHANISMS = %w(GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512 OAUTHBEARER)
+      SUPPORTED_MECHANISMS = %w(AWS_MSK_IAM GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512 OAUTHBEARER)
 
       def initialize(mechanism)
         unless SUPPORTED_MECHANISMS.include?(mechanism)

data/lib/kafka/protocol/sync_group_response.rb

@@ -13,9 +13,12 @@ module Kafka
       end
 
       def self.decode(decoder)
+        error_code = decoder.int16
+        member_assignment_bytes = decoder.bytes
+
         new(
-          error_code: decoder.int16,
-          member_assignment: MemberAssignment.decode(Decoder.from_string(decoder.bytes)),
+          error_code: error_code,
+          member_assignment: member_assignment_bytes ? MemberAssignment.decode(Decoder.from_string(member_assignment_bytes)) : nil
         )
       end
     end

data/lib/kafka/protocol/txn_offset_commit_response.rb

@@ -1,17 +1,46 @@
+# frozen_string_literal: true
+
 module Kafka
   module Protocol
     class TxnOffsetCommitResponse
+      class PartitionError
+        attr_reader :partition, :error_code
+
+        def initialize(partition:, error_code:)
+          @partition = partition
+          @error_code = error_code
+        end
+      end
+
+      class TopicPartitionsError
+        attr_reader :topic, :partitions
+
+        def initialize(topic:, partitions:)
+          @topic = topic
+          @partitions = partitions
+        end
+      end
 
-      attr_reader :error_code
+      attr_reader :errors
 
-      def initialize(error_code:)
-        @error_code = error_code
+      def initialize(errors:)
+        @errors = errors
       end
 
       def self.decode(decoder)
         _throttle_time_ms = decoder.int32
-        error_code = decoder.int16
-        new(error_code: error_code)
+        errors = decoder.array do
+          TopicPartitionsError.new(
+            topic: decoder.string,
+            partitions: decoder.array do
+              PartitionError.new(
+                partition: decoder.int32,
+                error_code: decoder.int16
+              )
+            end
+          )
+        end
+        new(errors: errors)
       end
     end
   end

data/lib/kafka/round_robin_assignment_strategy.rb

@@ -1,54 +1,52 @@
-# frozen_string_literal: true
-
-require "kafka/protocol/member_assignment"
-
 module Kafka
 
-  # A consumer group partition assignment strategy that assigns partitions to
-  # consumers in a round-robin fashion.
+  # A round robin assignment strategy inpired on the
+  # original java client round robin assignor. It's capable
+  # of handling identical as well as different topic subscriptions
+  # accross the same consumer group.
   class RoundRobinAssignmentStrategy
-    def initialize(cluster:)
-      @cluster = cluster
+    def protocol_name
+      "roundrobin"
     end
 
     # Assign the topic partitions to the group members.
     #
-    # @param members [Array<String>] member ids
-    # @param topics [Array<String>] topics
-    # @return [Hash<String, Protocol::MemberAssignment>] a hash mapping member
-    #   ids to assignments.
-    def assign(members:, topics:)
-      group_assignment = {}
-
-      members.each do |member_id|
-        group_assignment[member_id] = Protocol::MemberAssignment.new
-      end
-
-      topic_partitions = topics.flat_map do |topic|
-        begin
-          partitions = @cluster.partitions_for(topic).map(&:partition_id)
-        rescue UnknownTopicOrPartition
-          raise UnknownTopicOrPartition, "unknown topic #{topic}"
+    # @param cluster [Kafka::Cluster]
+    # @param members [Hash<String, Kafka::Protocol::JoinGroupResponse::Metadata>] a hash
+    #   mapping member ids to metadata
+    # @param partitions [Array<Kafka::ConsumerGroup::Assignor::Partition>] a list of
+    #   partitions the consumer group processes
+    # @return [Hash<String, Array<Kafka::ConsumerGroup::Assignor::Partition>] a hash
+    #   mapping member ids to partitions.
+    def call(cluster:, members:, partitions:)
+      partitions_per_member = Hash.new {|h, k| h[k] = [] }
+      relevant_partitions = valid_sorted_partitions(members, partitions)
+      members_ids = members.keys
+      iterator = (0...members.size).cycle
+      idx = iterator.next
+
+      relevant_partitions.each do |partition|
+        topic = partition.topic
+
+        while !members[members_ids[idx]].topics.include?(topic)
+          idx = iterator.next
         end
-        Array.new(partitions.count) { topic }.zip(partitions)
+
+        partitions_per_member[members_ids[idx]] << partition
+        idx = iterator.next
       end
 
-      partitions_per_member = topic_partitions.group_by.with_index do |_, index|
-        index % members.count
-      end.values
+      partitions_per_member
+    end
 
-      members.zip(partitions_per_member).each do |member_id, member_partitions|
-        unless member_partitions.nil?
-          member_partitions.each do |topic, partition|
-            group_assignment[member_id].assign(topic, [partition])
-          end
-        end
-      end
+    def valid_sorted_partitions(members, partitions)
+      subscribed_topics = members.map do |id, metadata|
+        metadata && metadata.topics
+      end.flatten.compact
 
-      group_assignment
-    rescue Kafka::LeaderNotAvailable
-      sleep 1
-      retry
+      partitions
+        .select { |partition| subscribed_topics.include?(partition.topic) }
+        .sort_by { |partition| partition.topic }
     end
   end
 end