ruby-kafka-custom 0.7.7.26

data/lib/kafka/protocol/describe_groups_request.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class DescribeGroupsRequest
+      def initialize(group_ids:)
+        @group_ids = group_ids
+      end
+
+      def api_key
+        DESCRIBE_GROUPS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::DescribeGroupsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@group_ids) { |group_id| encoder.write_string(group_id) }
+      end
+    end
+  end
+end

data/lib/kafka/protocol/describe_groups_response.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class DescribeGroupsResponse
+      class Group
+        attr_reader :error_code, :group_id, :state, :members, :protocol
+
+        def initialize(error_code:, group_id:, protocol_type:, protocol:, state:, members:)
+          @error_code = error_code
+          @group_id = group_id
+          @protocol_type = protocol_type
+          @protocol = protocol
+          @state = state
+          @members = members
+        end
+      end
+
+      class Member
+        attr_reader :member_id, :client_id, :client_host, :member_assignment
+
+        def initialize(member_id:, client_id:, client_host:, member_assignment:)
+          @member_id = member_id
+          @client_id = client_id
+          @client_host = client_host
+          @member_assignment = member_assignment
+        end
+      end
+
+      attr_reader :error_code, :groups
+
+      def initialize(groups:)
+        @groups = groups
+      end
+
+      def self.decode(decoder)
+        groups = decoder.array do
+          error_code = decoder.int16
+          group_id = decoder.string
+          state = decoder.string
+          protocol_type = decoder.string
+          protocol = decoder.string
+
+          members = decoder.array do
+            member_id = decoder.string
+            client_id = decoder.string
+            client_host = decoder.string
+            _metadata = decoder.bytes
+            assignment = MemberAssignment.decode(Decoder.from_string(decoder.bytes))
+
+            Member.new(
+              member_id: member_id,
+              client_id: client_id,
+              client_host: client_host,
+              member_assignment: assignment
+            )
+          end
+
+          Group.new(
+            error_code: error_code,
+            group_id: group_id,
+            state: state,
+            protocol_type: protocol_type,
+            protocol: protocol,
+            members: members
+          )
+        end
+
+        new(groups: groups)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/encoder.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Kafka
+  module Protocol
+    # An encoder wraps an IO object, making it easy to write specific data types
+    # to it.
+    class Encoder
+      # Initializes a new encoder.
+      #
+      # @param io [IO] an object that acts as an IO.
+      def initialize(io)
+        @io = io
+        @io.set_encoding(Encoding::BINARY)
+      end
+
+      # Writes bytes directly to the IO object.
+      #
+      # @param bytes [String]
+      # @return [nil]
+      def write(bytes)
+        @io.write(bytes)
+
+        nil
+      end
+
+      # Writes an 8-bit boolean to the IO object.
+      #
+      # @param boolean [Boolean]
+      # @return [nil]
+      def write_boolean(boolean)
+        boolean ? write_int8(1) : write_int8(0)
+      end
+
+      # Writes an 8-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
+      def write_int8(int)
+        write([int].pack("C"))
+      end
+
+      # Writes a 16-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
+      def write_int16(int)
+        write([int].pack("s>"))
+      end
+
+      # Writes a 32-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
+      def write_int32(int)
+        write([int].pack("l>"))
+      end
+
+      # Writes a 64-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
+      def write_int64(int)
+        write([int].pack("q>"))
+      end
+
+      # Writes an array to the IO object.
+      #
+      # Each item in the specified array will be yielded to the provided block;
+      # it's the responsibility of the block to write those items using the
+      # encoder.
+      #
+      # @param array [Array]
+      # @return [nil]
+      def write_array(array, &block)
+        if array.nil?
+          # An array can be null, which is different from it being empty.
+          write_int32(-1)
+        else
+          write_int32(array.size)
+          array.each(&block)
+        end
+      end
+
+      # Writes an array to the IO object.
+      # Just like #write_array, unless the size is under varint format
+      #
+      # @param array [Array]
+      # @return [nil]
+      def write_varint_array(array, &block)
+        if array.nil?
+          write_varint(-1)
+        else
+          write_varint(array.size)
+          array.each(&block)
+        end
+      end
+
+      # Writes a string to the IO object.
+      #
+      # @param string [String]
+      # @return [nil]
+      def write_string(string)
+        if string.nil?
+          write_int16(-1)
+        else
+          write_int16(string.bytesize)
+          write(string)
+        end
+      end
+
+      # Writes a string to the IO object, the size is under varint format
+      #
+      # @param string [String]
+      # @return [nil]
+      def write_varint_string(string)
+        if string.nil?
+          write_varint(-1)
+        else
+          write_varint(string.bytesize)
+          write(string)
+        end
+      end
+
+      # Writes an integer under varints serializing to the IO object.
+      # https://developers.google.com/protocol-buffers/docs/encoding#varints
+      #
+      # @param string [Integer]
+      # @return [nil]
+      def write_varint(int)
+        int = int << 1
+        int = ~int | 1 if int < 0
+
+        chunks = []
+        while int >> 7 != 0
+          chunks << (int & 0x7f | 0x80)
+          int >>= 7
+        end
+        chunks << int
+        write(chunks.pack("C*"))
+      end
+
+      # Writes a byte string to the IO object.
+      #
+      # @param bytes [String]
+      # @return [nil]
+      def write_bytes(bytes)
+        if bytes.nil?
+          write_int32(-1)
+        else
+          write_int32(bytes.bytesize)
+          write(bytes)
+        end
+      end
+
+      # Writes a byte string to the IO object, the size is under varint format
+      #
+      # @param bytes [String]
+      # @return [nil]
+      def write_varint_bytes(bytes)
+        if bytes.nil?
+          write_varint(-1)
+        else
+          write_varint(bytes.bytesize)
+          write(bytes)
+        end
+      end
+
+      # Encodes an object into a new buffer.
+      #
+      # @param object [#encode] the object that will encode itself.
+      # @return [String] the encoded data.
+      def self.encode_with(object)
+        buffer = StringIO.new
+        encoder = new(buffer)
+
+        object.encode(encoder)
+
+        buffer.string
+      end
+    end
+  end
+end

data/lib/kafka/protocol/end_txn_request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class EndTxnRequest
+      def initialize(transactional_id:, producer_id:, producer_epoch:, transaction_result:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @transaction_result = transaction_result
+      end
+
+      def api_key
+        END_TXN_API
+      end
+
+      def response_class
+        EndTxnResposne
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_boolean(@transaction_result)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/end_txn_response.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class EndTxnResposne
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/fetch_request.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    # A request to fetch messages from a given partition.
+    #
+    # ## API Specification
+    #
+    #     FetchRequest => ReplicaId MaxWaitTime MinBytes MaxBytes IsolationLevel [TopicName [Partition FetchOffset MaxBytes]]
+    #       ReplicaId => int32
+    #       MaxWaitTime => int32
+    #       MinBytes => int32
+    #       MaxBytes => int32
+    #       IsolationLevel => int8
+    #       TopicName => string
+    #       Partition => int32
+    #       FetchOffset => int64
+    #       MaxBytes => int32
+    #
+    class FetchRequest
+      ISOLATION_READ_UNCOMMITTED = 0
+      ISOLATION_READ_COMMITTED = 1
+
+      # @param max_wait_time [Integer]
+      # @param min_bytes [Integer]
+      # @param topics [Hash]
+      def initialize(max_wait_time:, min_bytes:, max_bytes:, topics:)
+        @replica_id = REPLICA_ID
+        @max_wait_time = max_wait_time
+        @min_bytes = min_bytes
+        @max_bytes = max_bytes
+        @topics = topics
+      end
+
+      def api_key
+        FETCH_API
+      end
+
+      def api_version
+        4
+      end
+
+      def response_class
+        Protocol::FetchResponse
+      end
+
+      def encode(encoder)
+        encoder.write_int32(@replica_id)
+        encoder.write_int32(@max_wait_time)
+        encoder.write_int32(@min_bytes)
+        encoder.write_int32(@max_bytes)
+        encoder.write_int8(ISOLATION_READ_COMMITTED)
+
+        encoder.write_array(@topics) do |topic, partitions|
+          encoder.write_string(topic)
+
+          encoder.write_array(partitions) do |partition, config|
+            fetch_offset = config.fetch(:fetch_offset)
+            max_bytes = config.fetch(:max_bytes)
+
+            encoder.write_int32(partition)
+            encoder.write_int64(fetch_offset)
+            encoder.write_int32(max_bytes)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafka/protocol/fetch_response.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "kafka/protocol/message_set"
+require "kafka/protocol/record_batch"
+
+module Kafka
+  module Protocol
+
+    # A response to a fetch request.
+    #
+    # ## API Specification
+    #
+    #     FetchResponse => ThrottleTimeMS [TopicName [Partition ErrorCode HighwaterMarkOffset LastStableOffset [AbortedTransaction] Records]]
+    #       ThrottleTimeMS => int32
+    #       TopicName => string
+    #       Partition => int32
+    #       ErrorCode => int16
+    #       HighwaterMarkOffset => int64
+    #       LastStableOffset => int64
+    #       MessageSetSize => int32
+    #       AbortedTransaction => [
+    #             ProducerId => int64
+    #             FirstOffset => int64
+    #       ]
+    #
+    class FetchResponse
+      MAGIC_BYTE_OFFSET = 16
+      MAGIC_BYTE_LENGTH = 1
+
+      class FetchedPartition
+        attr_reader :partition, :error_code
+        attr_reader :highwater_mark_offset, :last_stable_offset, :aborted_transactions, :messages
+
+        def initialize(partition:, error_code:, highwater_mark_offset:, last_stable_offset:, aborted_transactions:, messages:)
+          @partition = partition
+          @error_code = error_code
+          @highwater_mark_offset = highwater_mark_offset
+          @messages = messages
+          @last_stable_offset = last_stable_offset
+          @aborted_transactions = aborted_transactions
+        end
+      end
+
+      class FetchedTopic
+        attr_reader :name, :partitions
+
+        def initialize(name:, partitions:)
+          @name = name
+          @partitions = partitions
+        end
+      end
+
+      class AbortedTransaction
+        attr_reader :producer_id, :first_offset
+
+        def initialize(producer_id:, first_offset:)
+          @producer_id = producer_id
+          @first_offset = first_offset
+        end
+      end
+
+      attr_reader :topics
+
+      def initialize(topics: [], throttle_time_ms: 0)
+        @topics = topics
+        @throttle_time_ms = throttle_time_ms
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+
+        topics = decoder.array do
+          topic_name = decoder.string
+
+          partitions = decoder.array do
+            partition = decoder.int32
+            error_code = decoder.int16
+            highwater_mark_offset = decoder.int64
+            last_stable_offset = decoder.int64
+
+            aborted_transactions = decoder.array do
+              producer_id = decoder.int64
+              first_offset = decoder.int64
+              AbortedTransaction.new(
+                producer_id: producer_id,
+                first_offset: first_offset
+              )
+            end
+
+            messages_raw = decoder.bytes
+            messages = []
+
+            if !messages_raw.nil? && !messages_raw.empty?
+              messages_decoder = Decoder.from_string(messages_raw)
+
+              magic_byte = messages_decoder.peek(MAGIC_BYTE_OFFSET, MAGIC_BYTE_LENGTH)[0].to_i
+              if magic_byte == RecordBatch::MAGIC_BYTE
+                until messages_decoder.eof?
+                  begin
+                    record_batch = RecordBatch.decode(messages_decoder)
+                    messages << record_batch
+                  rescue InsufficientDataMessage
+                    if messages.length > 0
+                      break
+                    else
+                      raise
+                    end
+                  end
+                end
+              else
+                message_set = MessageSet.decode(messages_decoder)
+                messages << message_set
+              end
+            end
+
+            FetchedPartition.new(
+              partition: partition,
+              error_code: error_code,
+              highwater_mark_offset: highwater_mark_offset,
+              last_stable_offset: last_stable_offset,
+              aborted_transactions: aborted_transactions,
+              messages: messages
+            )
+          end
+
+          FetchedTopic.new(
+            name: topic_name,
+            partitions: partitions,
+          )
+        end
+
+        new(topics: topics, throttle_time_ms: throttle_time_ms)
+      end
+    end
+  end
+end