ruby-kafka-custom 0.7.7.26

data/lib/kafka/protocol/consumer_group_protocol.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class ConsumerGroupProtocol
+      def initialize(version: 0, topics:, user_data: nil)
+        @version = version
+        @topics = topics
+        @user_data = user_data
+      end
+
+      def encode(encoder)
+        encoder.write_int16(@version)
+        encoder.write_array(@topics) {|topic| encoder.write_string(topic) }
+        encoder.write_bytes(@user_data)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/create_partitions_request.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class CreatePartitionsRequest
+      def initialize(topics:, timeout:)
+        @topics, @timeout = topics, timeout
+      end
+
+      def api_key
+        CREATE_PARTITIONS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::CreatePartitionsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@topics) do |topic, count, assignments|
+          encoder.write_string(topic)
+          encoder.write_int32(count)
+          encoder.write_array(assignments) do |assignment|
+            encoder.write_array(assignment) do |broker|
+              encoder.write_int32(broker)
+            end
+          end
+        end
+        # Timeout is in ms.
+        encoder.write_int32(@timeout * 1000)
+        # validate_only. There isn't any use case for this in real life. So
+        # let's ignore it for now
+        encoder.write_boolean(false)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/create_partitions_response.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class CreatePartitionsResponse
+      attr_reader :errors
+
+      def initialize(throttle_time_ms:, errors:)
+        @throttle_time_ms = throttle_time_ms
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+        errors = decoder.array do
+          topic = decoder.string
+          error_code = decoder.int16
+          error_message = decoder.string
+          [topic, error_code, error_message]
+        end
+
+        new(throttle_time_ms: throttle_time_ms, errors: errors)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/create_topics_request.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class CreateTopicsRequest
+      def initialize(topics:, timeout:)
+        @topics, @timeout = topics, timeout
+      end
+
+      def api_key
+        CREATE_TOPICS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::CreateTopicsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@topics) do |topic, config|
+          encoder.write_string(topic)
+          encoder.write_int32(config.fetch(:num_partitions))
+          encoder.write_int16(config.fetch(:replication_factor))
+
+          # Replica assignments. We don't care.
+          encoder.write_array([])
+
+          encoder.write_array(config.fetch(:config)) do |config_name, config_value|
+            config_value = config_value.to_s unless config_value.nil?
+            encoder.write_string(config_name)
+            encoder.write_string(config_value)
+          end
+        end
+
+        # Timeout is in ms.
+        encoder.write_int32(@timeout * 1000)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/create_topics_response.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class CreateTopicsResponse
+      attr_reader :errors
+
+      def initialize(errors:)
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        errors = decoder.array do
+          topic = decoder.string
+          error_code = decoder.int16
+
+          [topic, error_code]
+        end
+
+        new(errors: errors)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/decoder.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    # A decoder wraps an IO object, making it easy to read specific data types
+    # from it. The Kafka protocol is not self-describing, so a client must call
+    # these methods in just the right order for things to work.
+    class Decoder
+      def self.from_string(str)
+        new(StringIO.new(str))
+      end
+
+      # Initializes a new decoder.
+      #
+      # @param io [IO] an object that acts as an IO.
+      def initialize(io)
+        @io = io
+      end
+
+      def eof?
+        @io.eof?
+      end
+
+      # Get some next bytes without touching the current io offset
+      #
+      # @return [Integer]
+      def peek(offset, length)
+        data = @io.read(offset + length)
+        return [] if data.nil?
+        @io.ungetc(data)
+        data.bytes[offset, offset + length] || []
+      end
+
+      # Decodes an 8-bit boolean from the IO object.
+      #
+      # @return [Boolean]
+      def boolean
+        read(1) == 0x1
+      end
+
+      # Decodes an 8-bit integer from the IO object.
+      #
+      # @return [Integer]
+      def int8
+        read(1).unpack("C").first
+      end
+
+      # Decodes a 16-bit integer from the IO object.
+      #
+      # @return [Integer]
+      def int16
+        read(2).unpack("s>").first
+      end
+
+      # Decodes a 32-bit integer from the IO object.
+      #
+      # @return [Integer]
+      def int32
+        read(4).unpack("l>").first
+      end
+
+      # Decodes a 64-bit integer from the IO object.
+      #
+      # @return [Integer]
+      def int64
+        read(8).unpack("q>").first
+      end
+
+      # Decodes an array from the IO object.
+      #
+      # The provided block will be called once for each item in the array. It is
+      # the responsibility of the block to decode the proper type in the block,
+      # since there's no information that allows the type to be inferred
+      # automatically.
+      #
+      # @return [Array]
+      def array(&block)
+        size = int32
+        size.times.map(&block)
+      end
+
+      # Decodes an array from the IO object.
+      # Just like #array except the size is in varint format
+      #
+      # @return [Array]
+      def varint_array(&block)
+        size = varint
+        size.times.map(&block)
+      end
+
+      # Decodes a string from the IO object.
+      #
+      # @return [String]
+      def string
+        size = int16
+
+        if size == -1
+          nil
+        else
+          read(size)
+        end
+      end
+
+      # Decodes a string from the IO object, the size is in varint format
+      #
+      # @return [String]
+      def varint_string
+        size = varint
+
+        if size == -1
+          nil
+        else
+          read(size)
+        end
+      end
+
+      # Read an integer under varints serializing from the IO object.
+      # https://developers.google.com/protocol-buffers/docs/encoding#varints
+      #
+      # @return [Integer]
+      def varint
+        group = 0
+        data = 0
+        while (chunk = int8) & 0x80 != 0
+          data |= (chunk & 0x7f) << group
+          group += 7
+        end
+        data |= chunk << group
+        data & 0b1 != 0 ? ~(data >> 1) : (data >> 1)
+      end
+
+      # Decodes a list of bytes from the IO object.
+      #
+      # @return [String]
+      def bytes
+        size = int32
+
+        if size == -1
+          nil
+        else
+          read(size)
+        end
+      end
+
+      # Decodes a list of bytes from the IO object. The size is in varint format
+      #
+      # @return [String]
+      def varint_bytes
+        size =  varint
+
+        if size == -1
+          nil
+        else
+          read(size)
+        end
+      end
+
+      # Reads the specified number of bytes from the IO object, returning them
+      # as a String.
+      #
+      # @return [String]
+      def read(number_of_bytes)
+        return "" if number_of_bytes == 0
+
+        data = @io.read(number_of_bytes) or raise EOFError
+
+        # If the `read` call returned less data than expected we should not
+        # proceed.
+        raise EOFError if data.size != number_of_bytes
+
+        data
+      end
+    end
+  end
+end

data/lib/kafka/protocol/delete_topics_request.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class DeleteTopicsRequest
+      def initialize(topics:, timeout:)
+        @topics, @timeout = topics, timeout
+      end
+
+      def api_key
+        DELETE_TOPICS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::DeleteTopicsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@topics) do |topic|
+          encoder.write_string(topic)
+        end
+        # Timeout is in ms.
+        encoder.write_int32(@timeout * 1000)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/delete_topics_response.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class DeleteTopicsResponse
+      attr_reader :errors
+
+      def initialize(errors:)
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        errors = decoder.array do
+          topic = decoder.string
+          error_code = decoder.int16
+
+          [topic, error_code]
+        end
+
+        new(errors: errors)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/describe_configs_request.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class DescribeConfigsRequest
+      def initialize(resources:)
+        @resources = resources
+      end
+
+      def api_key
+        DESCRIBE_CONFIGS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::DescribeConfigsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@resources) do |type, name, configs|
+          encoder.write_int8(type)
+          encoder.write_string(name)
+          encoder.write_array(configs) do |config|
+            encoder.write_string(config)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/describe_configs_response.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class DescribeConfigsResponse
+      class ResourceDescription
+        attr_reader :name, :type, :error_code, :error_message, :configs
+
+        def initialize(name:, type:, error_code:, error_message:, configs:)
+          @name = name
+          @type = type
+          @error_code = error_code
+          @error_message = error_message
+          @configs = configs
+        end
+      end
+
+      class ConfigEntry
+        attr_reader :name, :value, :read_only, :is_default, :is_sensitive
+
+        def initialize(name:, value:, read_only:, is_default:, is_sensitive:)
+          @name = name
+          @value = value
+          @read_only = read_only
+          @is_default = is_default
+          @is_sensitive = is_sensitive
+        end
+      end
+
+      attr_reader :resources
+
+      def initialize(throttle_time_ms:, resources:)
+        @throttle_time_ms = throttle_time_ms
+        @resources = resources
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+        resources = decoder.array do
+          error_code = decoder.int16
+          error_message = decoder.string
+
+          resource_type = decoder.int8
+          if Kafka::Protocol::RESOURCE_TYPES[resource_type].nil?
+            raise Kafka::ProtocolError, "Resource type not supported: #{resource_type}"
+          end
+          resource_name = decoder.string
+
+          configs = decoder.array do
+            ConfigEntry.new(
+              name: decoder.string,
+              value: decoder.string,
+              read_only: decoder.boolean,
+              is_default: decoder.boolean,
+              is_sensitive: decoder.boolean,
+            )
+          end
+
+          ResourceDescription.new(
+            type: RESOURCE_TYPES[resource_type],
+            name: resource_name,
+            error_code: error_code,
+            error_message: error_message,
+            configs: configs
+          )
+        end
+
+        new(throttle_time_ms: throttle_time_ms, resources: resources)
+      end
+    end
+
+  end
+end