ruby-kafka-custom 0.7.7.26

data/lib/kafka/protocol.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Kafka
+
+  # The protocol layer of the library.
+  #
+  # The Kafka protocol (https://kafka.apache.org/protocol) defines a set of API
+  # requests, each with a well-known numeric API key, as well as a set of error
+  # codes with specific meanings.
+  #
+  # This module, and the classes contained in it, implement the client side of
+  # the protocol.
+  module Protocol
+    # The replica id of non-brokers is always -1.
+    REPLICA_ID = -1
+
+    PRODUCE_API               = 0
+    FETCH_API                 = 1
+    LIST_OFFSET_API           = 2
+    TOPIC_METADATA_API        = 3
+    OFFSET_COMMIT_API         = 8
+    OFFSET_FETCH_API          = 9
+    FIND_COORDINATOR_API      = 10
+    JOIN_GROUP_API            = 11
+    HEARTBEAT_API             = 12
+    LEAVE_GROUP_API           = 13
+    SYNC_GROUP_API            = 14
+    DESCRIBE_GROUPS_API       = 15
+    LIST_GROUPS_API           = 16
+    SASL_HANDSHAKE_API        = 17
+    API_VERSIONS_API          = 18
+    CREATE_TOPICS_API         = 19
+    DELETE_TOPICS_API         = 20
+    INIT_PRODUCER_ID_API      = 22
+    ADD_PARTITIONS_TO_TXN_API = 24
+    END_TXN_API               = 26
+    DESCRIBE_CONFIGS_API      = 32
+    ALTER_CONFIGS_API         = 33
+    CREATE_PARTITIONS_API     = 37
+
+    # A mapping from numeric API keys to symbolic API names.
+    APIS = {
+      PRODUCE_API               => :produce,
+      FETCH_API                 => :fetch,
+      LIST_OFFSET_API           => :list_offset,
+      TOPIC_METADATA_API        => :topic_metadata,
+      OFFSET_COMMIT_API         => :offset_commit,
+      OFFSET_FETCH_API          => :offset_fetch,
+      FIND_COORDINATOR_API      => :find_coordinator,
+      JOIN_GROUP_API            => :join_group,
+      HEARTBEAT_API             => :heartbeat,
+      LEAVE_GROUP_API           => :leave_group,
+      SYNC_GROUP_API            => :sync_group,
+      SASL_HANDSHAKE_API        => :sasl_handshake,
+      API_VERSIONS_API          => :api_versions,
+      CREATE_TOPICS_API         => :create_topics,
+      DELETE_TOPICS_API         => :delete_topics,
+      INIT_PRODUCER_ID_API      => :init_producer_id_api,
+      ADD_PARTITIONS_TO_TXN_API => :add_partitions_to_txn_api,
+      END_TXN_API               => :end_txn_api,
+      DESCRIBE_CONFIGS_API      => :describe_configs_api,
+      CREATE_PARTITIONS_API     => :create_partitions
+    }
+
+    # A mapping from numeric error codes to exception classes.
+    ERRORS = {
+      -1 => UnknownError,
+       1 => OffsetOutOfRange,
+       2 => CorruptMessage,
+       3 => UnknownTopicOrPartition,
+       4 => InvalidMessageSize,
+       5 => LeaderNotAvailable,
+       6 => NotLeaderForPartition,
+       7 => RequestTimedOut,
+       8 => BrokerNotAvailable,
+       9 => ReplicaNotAvailable,
+      10 => MessageSizeTooLarge,
+      11 => StaleControllerEpoch,
+      12 => OffsetMetadataTooLarge,
+      13 => NetworkException,
+      14 => CoordinatorLoadInProgress,
+      15 => CoordinatorNotAvailable,
+      16 => NotCoordinatorForGroup,
+      17 => InvalidTopic,
+      18 => RecordListTooLarge,
+      19 => NotEnoughReplicas,
+      20 => NotEnoughReplicasAfterAppend,
+      21 => InvalidRequiredAcks,
+      22 => IllegalGeneration,
+      23 => InconsistentGroupProtocol,
+      24 => InvalidGroupId,
+      25 => UnknownMemberId,
+      26 => InvalidSessionTimeout,
+      27 => RebalanceInProgress,
+      28 => InvalidCommitOffsetSize,
+      29 => TopicAuthorizationFailed,
+      30 => GroupAuthorizationFailed,
+      31 => ClusterAuthorizationFailed,
+      32 => InvalidTimestamp,
+      33 => UnsupportedSaslMechanism,
+      34 => InvalidSaslState,
+      35 => UnsupportedVersion,
+      36 => TopicAlreadyExists,
+      37 => InvalidPartitions,
+      38 => InvalidReplicationFactor,
+      39 => InvalidReplicaAssignment,
+      40 => InvalidConfig,
+      41 => NotController,
+      42 => InvalidRequest,
+      43 => UnsupportedForMessageFormat,
+      44 => PolicyViolation,
+      45 => OutOfOrderSequenceNumberError,
+      46 => DuplicateSequenceNumberError,
+      47 => InvalidProducerEpochError,
+      48 => InvalidTxnStateError,
+      49 => InvalidProducerIDMappingError,
+      50 => InvalidTransactionTimeoutError,
+      51 => ConcurrentTransactionError,
+      52 => TransactionCoordinatorFencedError
+    }
+
+    # A mapping from int to corresponding resource type in symbol.
+    # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
+    RESOURCE_TYPE_UNKNOWN          = 0
+    RESOURCE_TYPE_ANY              = 1
+    RESOURCE_TYPE_TOPIC            = 2
+    RESOURCE_TYPE_GROUP            = 3
+    RESOURCE_TYPE_CLUSTER          = 4
+    RESOURCE_TYPE_TRANSACTIONAL_ID = 5
+    RESOURCE_TYPE_DELEGATION_TOKEN = 6
+    RESOURCE_TYPES = {
+      RESOURCE_TYPE_UNKNOWN          => :unknown,
+      RESOURCE_TYPE_ANY              => :any,
+      RESOURCE_TYPE_TOPIC            => :topic,
+      RESOURCE_TYPE_GROUP            => :group,
+      RESOURCE_TYPE_CLUSTER          => :cluster,
+      RESOURCE_TYPE_TRANSACTIONAL_ID => :transactional_id,
+      RESOURCE_TYPE_DELEGATION_TOKEN => :delegation_token,
+    }
+
+    # Coordinator types. Since Kafka 0.11.0, there are types of coordinators:
+    # Group and Transaction
+    COORDINATOR_TYPE_GROUP = 0
+    COORDINATOR_TYPE_TRANSACTION = 1
+
+    # Handles an error code by either doing nothing (if there was no error) or
+    # by raising an appropriate exception.
+    #
+    # @param error_code Integer
+    # @raise [ProtocolError]
+    # @return [nil]
+    def self.handle_error(error_code, error_message = nil)
+      if error_code == 0
+        # No errors, yay!
+      elsif error = ERRORS[error_code]
+        raise error, error_message
+      else
+        raise UnknownError, "Unknown error with code #{error_code} #{error_message}"
+      end
+    end
+
+    # Returns the symbolic name for an API key.
+    #
+    # @param api_key Integer
+    # @return [Symbol]
+    def self.api_name(api_key)
+      APIS.fetch(api_key, :unknown)
+    end
+  end
+end
+
+require "kafka/protocol/metadata_request"
+require "kafka/protocol/metadata_response"
+require "kafka/protocol/produce_request"
+require "kafka/protocol/produce_response"
+require "kafka/protocol/fetch_request"
+require "kafka/protocol/fetch_response"
+require "kafka/protocol/list_offset_request"
+require "kafka/protocol/list_offset_response"
+require "kafka/protocol/find_coordinator_request"
+require "kafka/protocol/find_coordinator_response"
+require "kafka/protocol/join_group_request"
+require "kafka/protocol/join_group_response"
+require "kafka/protocol/sync_group_request"
+require "kafka/protocol/sync_group_response"
+require "kafka/protocol/leave_group_request"
+require "kafka/protocol/leave_group_response"
+require "kafka/protocol/heartbeat_request"
+require "kafka/protocol/heartbeat_response"
+require "kafka/protocol/offset_fetch_request"
+require "kafka/protocol/offset_fetch_response"
+require "kafka/protocol/offset_commit_request"
+require "kafka/protocol/offset_commit_response"
+require "kafka/protocol/api_versions_request"
+require "kafka/protocol/api_versions_response"
+require "kafka/protocol/sasl_handshake_request"
+require "kafka/protocol/sasl_handshake_response"
+require "kafka/protocol/create_topics_request"
+require "kafka/protocol/create_topics_response"
+require "kafka/protocol/delete_topics_request"
+require "kafka/protocol/delete_topics_response"
+require "kafka/protocol/describe_configs_request"
+require "kafka/protocol/describe_configs_response"
+require "kafka/protocol/alter_configs_request"
+require "kafka/protocol/alter_configs_response"
+require "kafka/protocol/create_partitions_request"
+require "kafka/protocol/create_partitions_response"
+require "kafka/protocol/list_groups_request"
+require "kafka/protocol/list_groups_response"
+require "kafka/protocol/describe_groups_request"
+require "kafka/protocol/describe_groups_response"
+require "kafka/protocol/init_producer_id_request"
+require "kafka/protocol/init_producer_id_response"
+require "kafka/protocol/add_partitions_to_txn_request"
+require "kafka/protocol/add_partitions_to_txn_response"
+require "kafka/protocol/end_txn_request"
+require "kafka/protocol/end_txn_response"

data/lib/kafka/protocol/add_partitions_to_txn_request.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddPartitionsToTxnRequest
+      def initialize(transactional_id: nil, producer_id:, producer_epoch:, topics:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @topics = topics
+      end
+
+      def api_key
+        ADD_PARTITIONS_TO_TXN_API
+      end
+
+      def response_class
+        AddPartitionsToTxnResponse
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_array(@topics.to_a) do |topic, partitions|
+          encoder.write_string(topic)
+          encoder.write_array(partitions) do |partition|
+            encoder.write_int32(partition)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafka/protocol/add_partitions_to_txn_response.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddPartitionsToTxnResponse
+      class PartitionError
+        attr_reader :partition, :error_code
+
+        def initialize(partition:, error_code:)
+          @partition = partition
+          @error_code = error_code
+        end
+      end
+
+      class TopicPartitionsError
+        attr_reader :topic, :partitions
+
+        def initialize(topic:, partitions:)
+          @topic = topic
+          @partitions = partitions
+        end
+      end
+
+      attr_reader :errors
+
+      def initialize(errors:)
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        errors = decoder.array do
+          TopicPartitionsError.new(
+            topic: decoder.string,
+            partitions: decoder.array do
+              PartitionError.new(
+                partition: decoder.int32,
+                error_code: decoder.int16
+              )
+            end
+          )
+        end
+        new(errors: errors)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/alter_configs_request.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class AlterConfigsRequest
+      def initialize(resources:)
+        @resources = resources
+      end
+
+      def api_key
+        ALTER_CONFIGS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::AlterConfigsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@resources) do |type, name, configs|
+          encoder.write_int8(type)
+          encoder.write_string(name)
+
+          configs = configs.to_a
+          encoder.write_array(configs) do |config_name, config_value|
+            # Config value is nullable. In other cases, we must write the
+            # stringified value.
+            config_value = config_value.to_s unless config_value.nil?
+
+            encoder.write_string(config_name)
+            encoder.write_string(config_value)
+          end
+        end
+        # validate_only. We'll skip this feature.
+        encoder.write_boolean(false)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/alter_configs_response.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AlterConfigsResponse
+      class ResourceDescription
+        attr_reader :name, :type, :error_code, :error_message
+
+        def initialize(name:, type:, error_code:, error_message:)
+          @name = name
+          @type = type
+          @error_code = error_code
+          @error_message = error_message
+        end
+      end
+
+      attr_reader :resources
+
+      def initialize(throttle_time_ms:, resources:)
+        @throttle_time_ms = throttle_time_ms
+        @resources = resources
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+        resources = decoder.array do
+          error_code = decoder.int16
+          error_message = decoder.string
+
+          resource_type = decoder.int8
+          if Kafka::Protocol::RESOURCE_TYPES[resource_type].nil?
+            raise Kafka::ProtocolError, "Resource type not supported: #{resource_type}"
+          end
+          resource_name = decoder.string
+
+          ResourceDescription.new(
+            type: RESOURCE_TYPES[resource_type],
+            name: resource_name,
+            error_code: error_code,
+            error_message: error_message
+          )
+        end
+
+        new(throttle_time_ms: throttle_time_ms, resources: resources)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/api_versions_request.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class ApiVersionsRequest
+      def api_key
+        API_VERSIONS_API
+      end
+
+      def encode(encoder)
+        # Nothing to do.
+      end
+
+      def response_class
+        Protocol::ApiVersionsResponse
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/api_versions_response.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+
+    class ApiVersionsResponse
+      class ApiInfo
+        attr_reader :api_key, :min_version, :max_version
+
+        def initialize(api_key:, min_version:, max_version:)
+          @api_key, @min_version, @max_version = api_key, min_version, max_version
+        end
+
+        def api_name
+          Protocol.api_name(api_key)
+        end
+
+        def version_supported?(version)
+          (min_version..max_version).include?(version)
+        end
+
+        def to_s
+          "#{api_name}=#{min_version}..#{max_version}"
+        end
+
+        def inspect
+          "#<Kafka api version #{to_s}>"
+        end
+      end
+
+      attr_reader :error_code, :apis
+
+      def initialize(error_code:, apis:)
+        @error_code = error_code
+        @apis = apis
+      end
+
+      def self.decode(decoder)
+        error_code = decoder.int16
+
+        apis = decoder.array do
+          ApiInfo.new(
+            api_key: decoder.int16,
+            min_version: decoder.int16,
+            max_version: decoder.int16,
+          )
+        end
+
+        new(error_code: error_code, apis: apis)
+      end
+    end
+  end
+end