ruby-kafka 0.7.0 → 0.7.1.beta1

data/lib/kafka/protocol.rb

@@ -14,46 +14,52 @@ module Kafka
     # The replica id of non-brokers is always -1.
     REPLICA_ID = -1
 
-    PRODUCE_API = 0
-    FETCH_API = 1
-    LIST_OFFSET_API = 2
-    TOPIC_METADATA_API = 3
-    OFFSET_COMMIT_API = 8
-    OFFSET_FETCH_API = 9
-    FIND_COORDINATOR_API = 10
-    JOIN_GROUP_API = 11
-    HEARTBEAT_API = 12
-    LEAVE_GROUP_API = 13
-    SYNC_GROUP_API = 14
-    DESCRIBE_GROUPS_API = 15
-    LIST_GROUPS_API = 16
-    SASL_HANDSHAKE_API = 17
-    API_VERSIONS_API = 18
-    CREATE_TOPICS_API = 19
-    DELETE_TOPICS_API = 20
-    DESCRIBE_CONFIGS_API = 32
-    ALTER_CONFIGS_API = 33
-    CREATE_PARTITIONS_API = 37
+    PRODUCE_API               = 0
+    FETCH_API                 = 1
+    LIST_OFFSET_API           = 2
+    TOPIC_METADATA_API        = 3
+    OFFSET_COMMIT_API         = 8
+    OFFSET_FETCH_API          = 9
+    FIND_COORDINATOR_API      = 10
+    JOIN_GROUP_API            = 11
+    HEARTBEAT_API             = 12
+    LEAVE_GROUP_API           = 13
+    SYNC_GROUP_API            = 14
+    DESCRIBE_GROUPS_API       = 15
+    LIST_GROUPS_API           = 16
+    SASL_HANDSHAKE_API        = 17
+    API_VERSIONS_API          = 18
+    CREATE_TOPICS_API         = 19
+    DELETE_TOPICS_API         = 20
+    INIT_PRODUCER_ID_API      = 22
+    ADD_PARTITIONS_TO_TXN_API = 24
+    END_TXN_API               = 26
+    DESCRIBE_CONFIGS_API      = 32
+    ALTER_CONFIGS_API         = 33
+    CREATE_PARTITIONS_API     = 37
 
     # A mapping from numeric API keys to symbolic API names.
     APIS = {
-      PRODUCE_API => :produce,
-      FETCH_API => :fetch,
-      LIST_OFFSET_API => :list_offset,
-      TOPIC_METADATA_API => :topic_metadata,
-      OFFSET_COMMIT_API => :offset_commit,
-      OFFSET_FETCH_API => :offset_fetch,
-      FIND_COORDINATOR_API => :find_coordinator,
-      JOIN_GROUP_API => :join_group,
-      HEARTBEAT_API => :heartbeat,
-      LEAVE_GROUP_API => :leave_group,
-      SYNC_GROUP_API => :sync_group,
-      SASL_HANDSHAKE_API => :sasl_handshake,
-      API_VERSIONS_API => :api_versions,
-      CREATE_TOPICS_API => :create_topics,
-      DELETE_TOPICS_API => :delete_topics,
-      DESCRIBE_CONFIGS_API => :describe_configs_api,
-      CREATE_PARTITIONS_API => :create_partitions
+      PRODUCE_API               => :produce,
+      FETCH_API                 => :fetch,
+      LIST_OFFSET_API           => :list_offset,
+      TOPIC_METADATA_API        => :topic_metadata,
+      OFFSET_COMMIT_API         => :offset_commit,
+      OFFSET_FETCH_API          => :offset_fetch,
+      FIND_COORDINATOR_API      => :find_coordinator,
+      JOIN_GROUP_API            => :join_group,
+      HEARTBEAT_API             => :heartbeat,
+      LEAVE_GROUP_API           => :leave_group,
+      SYNC_GROUP_API            => :sync_group,
+      SASL_HANDSHAKE_API        => :sasl_handshake,
+      API_VERSIONS_API          => :api_versions,
+      CREATE_TOPICS_API         => :create_topics,
+      DELETE_TOPICS_API         => :delete_topics,
+      INIT_PRODUCER_ID_API      => :init_producer_id_api,
+      ADD_PARTITIONS_TO_TXN_API => :add_partitions_to_txn_api,
+      END_TXN_API               => :end_txn_api,
+      DESCRIBE_CONFIGS_API      => :describe_configs_api,
+      CREATE_PARTITIONS_API     => :create_partitions
     }
 
     # A mapping from numeric error codes to exception classes.
@@ -95,24 +101,32 @@ module Kafka
       39 => InvalidReplicaAssignment,
       40 => InvalidConfig,
       41 => NotController,
-      42 => InvalidRequest
+      42 => InvalidRequest,
+      45 => OutOfOrderSequenceNumberError,
+      46 => DuplicateSequenceNumberError,
+      47 => InvalidProducerEpochError,
+      48 => InvalidTxnStateError,
+      49 => InvalidProducerIDMappingError,
+      50 => InvalidTransactionTimeoutError,
+      51 => ConcurrentTransactionError,
+      52 => TransactionCoordinatorFencedError
     }
 
     # A mapping from int to corresponding resource type in symbol.
     # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
-    RESOURCE_TYPE_UNKNOWN = 0
-    RESOURCE_TYPE_ANY = 1
-    RESOURCE_TYPE_TOPIC = 2
-    RESOURCE_TYPE_GROUP = 3
-    RESOURCE_TYPE_CLUSTER = 4
+    RESOURCE_TYPE_UNKNOWN          = 0
+    RESOURCE_TYPE_ANY              = 1
+    RESOURCE_TYPE_TOPIC            = 2
+    RESOURCE_TYPE_GROUP            = 3
+    RESOURCE_TYPE_CLUSTER          = 4
     RESOURCE_TYPE_TRANSACTIONAL_ID = 5
     RESOURCE_TYPE_DELEGATION_TOKEN = 6
     RESOURCE_TYPES = {
-      RESOURCE_TYPE_UNKNOWN => :unknown,
-      RESOURCE_TYPE_ANY => :any,
-      RESOURCE_TYPE_TOPIC => :topic,
-      RESOURCE_TYPE_GROUP => :group,
-      RESOURCE_TYPE_CLUSTER => :cluster,
+      RESOURCE_TYPE_UNKNOWN          => :unknown,
+      RESOURCE_TYPE_ANY              => :any,
+      RESOURCE_TYPE_TOPIC            => :topic,
+      RESOURCE_TYPE_GROUP            => :group,
+      RESOURCE_TYPE_CLUSTER          => :cluster,
       RESOURCE_TYPE_TRANSACTIONAL_ID => :transactional_id,
       RESOURCE_TYPE_DELEGATION_TOKEN => :delegation_token,
     }
@@ -188,3 +202,9 @@ require "kafka/protocol/list_groups_request"
 require "kafka/protocol/list_groups_response"
 require "kafka/protocol/describe_groups_request"
 require "kafka/protocol/describe_groups_response"
+require "kafka/protocol/init_producer_id_request"
+require "kafka/protocol/init_producer_id_response"
+require "kafka/protocol/add_partitions_to_txn_request"
+require "kafka/protocol/add_partitions_to_txn_response"
+require "kafka/protocol/end_txn_request"
+require "kafka/protocol/end_txn_response"

data/lib/kafka/protocol/add_partitions_to_txn_request.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddPartitionsToTxnRequest
+      def initialize(transactional_id: nil, producer_id:, producer_epoch:, topics:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @topics = topics
+      end
+
+      def api_key
+        ADD_PARTITIONS_TO_TXN_API
+      end
+
+      def response_class
+        AddPartitionsToTxnResponse
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_array(@topics.to_a) do |topic, partitions|
+          encoder.write_string(topic)
+          encoder.write_array(partitions) do |partition|
+            encoder.write_int32(partition)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafka/protocol/add_partitions_to_txn_response.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddPartitionsToTxnResponse
+      class PartitionError
+        attr_reader :partition, :error_code
+
+        def initialize(partition:, error_code:)
+          @partition = partition
+          @error_code = error_code
+        end
+      end
+
+      class TopicPartitionsError
+        attr_reader :topic, :partitions
+
+        def initialize(topic:, partitions:)
+          @topic = topic
+          @partitions = partitions
+        end
+      end
+
+      attr_reader :errors
+
+      def initialize(errors:)
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        errors = decoder.array do
+          TopicPartitionsError.new(
+            topic: decoder.string,
+            partitions: decoder.array do
+              PartitionError.new(
+                partition: decoder.int32,
+                error_code: decoder.int16
+              )
+            end
+          )
+        end
+        new(errors: errors)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/decoder.rb

@@ -6,8 +6,6 @@ module Kafka
     # from it. The Kafka protocol is not self-describing, so a client must call
     # these methods in just the right order for things to work.
     class Decoder
-      VARINT_MASK = 0b10000000
-
       def self.from_string(str)
         new(StringIO.new(str))
       end
@@ -123,12 +121,11 @@ module Kafka
       def varint
         group = 0
         data = 0
-        loop do
-          chunk = int8
-          data |= (chunk & (~VARINT_MASK)) << group
+        while (chunk = int8) & 0x80 != 0
+          data |= (chunk & 0x7f) << group
           group += 7
-          break if (chunk & VARINT_MASK) == 0
         end
+        data |= chunk << group
         data & 0b1 != 0 ? ~(data >> 1) : (data >> 1)
       end
 

data/lib/kafka/protocol/encoder.rb

@@ -7,8 +7,6 @@ module Kafka
     # An encoder wraps an IO object, making it easy to write specific data types
     # to it.
     class Encoder
-      VARINT_MASK = 0b10000000
-
       # Initializes a new encoder.
       #
       # @param io [IO] an object that acts as an IO.
@@ -134,16 +132,13 @@ module Kafka
         int = int << 1
         int = ~int | 1 if int < 0
 
-        loop do
-          chunk = int & (~VARINT_MASK)
-          int = int >> 7
-          if int == 0
-            write_int8(chunk)
-            return
-          else
-            write_int8(chunk | VARINT_MASK)
-          end
+        chunks = []
+        while int & 0xff80 != 0
+          chunks << (int & 0x7f | 0x80)
+          int >>= 7
         end
+        chunks << int
+        write(chunks.pack("C*"))
       end
 
       # Writes a byte string to the IO object.

data/lib/kafka/protocol/end_txn_request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class EndTxnRequest
+      def initialize(transactional_id:, producer_id:, producer_epoch:, transaction_result:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @transaction_result = transaction_result
+      end
+
+      def api_key
+        END_TXN_API
+      end
+
+      def response_class
+        EndTxnResposne
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_boolean(@transaction_result)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/end_txn_response.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class EndTxnResposne
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/fetch_request.rb

@@ -7,10 +7,12 @@ module Kafka
     #
     # ## API Specification
     #
-    #     FetchRequest => ReplicaId MaxWaitTime MinBytes [TopicName [Partition FetchOffset MaxBytes]]
+    #     FetchRequest => ReplicaId MaxWaitTime MinBytes MaxBytes IsolationLevel [TopicName [Partition FetchOffset MaxBytes]]
     #       ReplicaId => int32
     #       MaxWaitTime => int32
     #       MinBytes => int32
+    #       MaxBytes => int32
+    #       IsolationLevel => int8
     #       TopicName => string
     #       Partition => int32
     #       FetchOffset => int64

data/lib/kafka/protocol/fetch_response.rb

@@ -10,12 +10,18 @@ module Kafka
     #
     # ## API Specification
     #
-    #     FetchResponse => [TopicName [Partition ErrorCode HighwaterMarkOffset MessageSetSize MessageSet]]
+    #     FetchResponse => ThrottleTimeMS [TopicName [Partition ErrorCode HighwaterMarkOffset LastStableOffset [AbortedTransaction] Records]]
+    #       ThrottleTimeMS => int32
     #       TopicName => string
     #       Partition => int32
     #       ErrorCode => int16
     #       HighwaterMarkOffset => int64
+    #       LastStableOffset => int64
     #       MessageSetSize => int32
+    #       AbortedTransaction => [
+    #             ProducerId => int64
+    #             FirstOffset => int64
+    #       ]
     #
     class FetchResponse
       MAGIC_BYTE_OFFSET = 16
@@ -44,6 +50,15 @@ module Kafka
         end
       end
 
+      class AbortedTransaction
+        attr_reader :producer_id, :first_offset
+
+        def initialize(producer_id:, first_offset:)
+          @producer_id = producer_id
+          @first_offset = first_offset
+        end
+      end
+
       attr_reader :topics
 
       def initialize(topics: [], throttle_time_ms: 0)
@@ -66,32 +81,36 @@ module Kafka
             aborted_transactions = decoder.array do
               producer_id = decoder.int64
               first_offset = decoder.int64
-              {
+              AbortedTransaction.new(
                 producer_id: producer_id,
                 first_offset: first_offset
-              }
+              )
             end
 
-            messages_decoder = Decoder.from_string(decoder.bytes)
+            messages_raw = decoder.bytes
             messages = []
 
-            magic_byte = messages_decoder.peek(MAGIC_BYTE_OFFSET, MAGIC_BYTE_LENGTH)[0].to_i
-            if magic_byte == RecordBatch::MAGIC_BYTE
-              until messages_decoder.eof?
-                begin
-                  record_batch = RecordBatch.decode(messages_decoder)
-                  messages += record_batch.records
-                rescue InsufficientDataMessage
-                  if messages.length > 0
-                    break
-                  else
-                    raise
+            if !messages_raw.nil? && !messages_raw.empty?
+              messages_decoder = Decoder.from_string(messages_raw)
+
+              magic_byte = messages_decoder.peek(MAGIC_BYTE_OFFSET, MAGIC_BYTE_LENGTH)[0].to_i
+              if magic_byte == RecordBatch::MAGIC_BYTE
+                until messages_decoder.eof?
+                  begin
+                    record_batch = RecordBatch.decode(messages_decoder)
+                    messages << record_batch
+                  rescue InsufficientDataMessage
+                    if messages.length > 0
+                      break
+                    else
+                      raise
+                    end
                   end
                 end
+              else
+                message_set = MessageSet.decode(messages_decoder)
+                messages << message_set
               end
-            else
-              message_set = MessageSet.decode(messages_decoder)
-              messages = message_set.messages
             end
 
             FetchedPartition.new(

data/lib/kafka/protocol/init_producer_id_request.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class InitProducerIDRequest
+      def initialize(transactional_id: nil, transactional_timeout:)
+        @transactional_id = transactional_id
+        @transactional_timeout = transactional_timeout
+      end
+
+      def api_key
+        INIT_PRODUCER_ID_API
+      end
+
+      def response_class
+        InitProducerIDResponse
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id)
+        # Timeout is in ms unit
+        encoder.write_int32(@transactional_timeout * 1000)
+      end
+    end
+  end
+end