ruby-kafka 0.7.0.alpha1 → 0.7.0.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a913d7cca1f36c5d37398c39620db551399b18073ae1cc11eb110e0d32f8a39
-  data.tar.gz: 5ccef5d293d0d8732eb592496468f8f7840a4a8a62ca1982c0c81d7b224a2f93
+  metadata.gz: 2ddcaacf16a7990d53463e5302a389cae33c7782d279ada36a7b32d229a302a5
+  data.tar.gz: 872a5d08ec43a2e4ad550b362bacbcf3c5a523de8bcea49ebc240086b7bba74d
 SHA512:
-  metadata.gz: e21b28dd124d83ca1c470f807e21ceb1d4c117a6aa9f22103f405752f4212fc5119d969b09e58a6bd35dfe56ea7611274dd79cb7c10fe8c573ce6031b3a8a547
-  data.tar.gz: dab421eb3da87aa62e134e16a0cafb73825df1ca09b7dd3c702d12e9235c303ec286b581b4f5c87fafc95ae19983d76db3995fb1eda5ef922ba1bae4f224bbbc
+  metadata.gz: 4352915740c56ff947cf91e951a9298f39a78db7c51421456a8db2c037f642ab6f791bfe3eb1bbeff01700d573e342a020b0ed7896c481996cf59d6e21d3b4f2
+  data.tar.gz: 8dec7bfdb51ff8321c176b29c6a3cf378b837c807aaf050b8cefe8ce5224044f016f16234f9dfae5104946882fe7817d69650206219e6640f44738c60bff682d

data/.circleci/config.yml

@@ -11,38 +11,6 @@ jobs:
       - run: bundle exec rspec
       - run: bundle exec rubocop
 
-  kafka-0.10:
-    docker:
-      - image: circleci/ruby:2.4.1-node
-        environment:
-          LOG_LEVEL: DEBUG
-      - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:0.10.2.1
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:0.10.2.1
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9093
-          KAFKA_PORT: 9093
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:0.10.2.1
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9094
-          KAFKA_PORT: 9094
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-    steps:
-      - checkout
-      - run: bundle install --path vendor/bundle
-      - run: bundle exec rspec --profile --tag functional spec/functional
-
   kafka-0.11:
     docker:
       - image: circleci/ruby:2.4.1-node
@@ -110,6 +78,5 @@ workflows:
   test:
     jobs:
       - unit
-      - kafka-0.10
       - kafka-0.11
       - kafka-1.0

data/lib/kafka.rb

@@ -42,6 +42,10 @@ module Kafka
   class CorruptMessage < ProtocolError
   end
 
+  # When the record array length doesn't match real number of received records
+  class InsufficientDataMessage < Error
+  end
+
   class UnknownError < ProtocolError
   end
 

data/lib/kafka/compressor.rb

@@ -30,13 +30,31 @@ module Kafka
       @instrumenter = instrumenter
     end
 
-    # @param message_set [Protocol::MessageSet]
+    # @param record_batch [Protocol::RecordBatch]
     # @param offset [Integer] used to simulate broker behaviour in tests
-    # @return [Protocol::MessageSet]
-    def compress(message_set, offset: -1)
+    # @return [Protocol::RecordBatch]
+    def compress(record_batch, offset: -1)
+      if record_batch.is_a?(Protocol::RecordBatch)
+        compress_record_batch(record_batch)
+      else
+        # Deprecated message set format
+        compress_message_set(record_batch, offset)
+      end
+    end
+
+    private
+
+    def compress_message_set(message_set, offset)
       return message_set if @codec.nil? || message_set.size < @threshold
 
-      compressed_data = compress_data(message_set)
+      data = Protocol::Encoder.encode_with(message_set)
+      compressed_data = @codec.compress(data)
+
+      @instrumenter.instrument("compress.compressor") do |notification|
+        notification[:message_count] = message_set.size
+        notification[:uncompressed_bytesize] = data.bytesize
+        notification[:compressed_bytesize] = compressed_data.bytesize
+      end
 
       wrapper_message = Protocol::Message.new(
         value: compressed_data,
@@ -47,20 +65,21 @@ module Kafka
       Protocol::MessageSet.new(messages: [wrapper_message])
     end
 
-    private
+    def compress_record_batch(record_batch)
+      if @codec.nil? || record_batch.size < @threshold
+        record_batch.codec_id = 0
+        return Protocol::Encoder.encode_with(record_batch)
+      end
 
-    def compress_data(message_set)
-      data = Protocol::Encoder.encode_with(message_set)
+      record_batch.codec_id = @codec.codec_id
+      data = Protocol::Encoder.encode_with(record_batch)
 
       @instrumenter.instrument("compress.compressor") do |notification|
-        compressed_data = @codec.compress(data)
-
-        notification[:message_count] = message_set.size
-        notification[:uncompressed_bytesize] = data.bytesize
-        notification[:compressed_bytesize] = compressed_data.bytesize
-
-        compressed_data
+        notification[:message_count] = record_batch.size
+        notification[:compressed_bytesize] = data.bytesize
       end
+
+      data
     end
   end
 end

data/lib/kafka/consumer.rb

@@ -221,7 +221,7 @@ module Kafka
 
             @instrumenter.instrument("process_message.consumer", notification) do
               begin
-                yield message
+                yield message unless message.is_control_record
                 @current_offsets[message.topic][message.partition] = message.offset
               rescue => e
                 location = "#{message.topic}/#{message.partition} at offset #{message.offset}"
@@ -289,6 +289,9 @@ module Kafka
 
         batches.each do |batch|
           unless batch.empty?
+            raw_messages = batch.messages
+            batch.messages = raw_messages.reject(&:is_control_record)
+
             notification = {
               topic: batch.topic,
               partition: batch.partition,
@@ -314,9 +317,10 @@ module Kafka
                 @logger.error "Exception raised when processing #{location} -- #{e.class}: #{e}\n#{backtrace}"
 
                 raise ProcessingError.new(batch.topic, batch.partition, offset_range)
+              ensure
+                batch.messages = raw_messages
               end
             end
-
             mark_message_as_processed(batch.messages.last) if automatically_mark_as_processed
 
             # We've successfully processed a batch from the partition, so we can clear

data/lib/kafka/fetched_batch.rb

@@ -14,7 +14,7 @@ module Kafka
     attr_reader :highwater_mark_offset
 
     # @return [Array<Kafka::FetchedMessage>]
-    attr_reader :messages
+    attr_accessor :messages
 
     def initialize(topic:, partition:, highwater_mark_offset:, messages:)
       @topic = topic

data/lib/kafka/fetched_message.rb

@@ -33,5 +33,10 @@ module Kafka
     def create_time
       @message.create_time
     end
+
+    # @return [Boolean] whether this record is a control record
+    def is_control_record
+      @message.is_control_record
+    end
   end
 end

data/lib/kafka/message_buffer.rb

@@ -17,7 +17,7 @@ module Kafka
     end
 
     def write(value:, key:, topic:, partition:, create_time: Time.now)
-      message = Protocol::Message.new(key: key, value: value, create_time: create_time)
+      message = Protocol::Record.new(key: key, value: value, create_time: create_time)
 
       buffer_for(topic, partition) << message
 

data/lib/kafka/produce_operation.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "kafka/protocol/message_set"
+require "kafka/protocol/record_batch"
 
 module Kafka
   # A produce operation attempts to send all messages in a buffer to the Kafka cluster.
@@ -86,12 +87,10 @@ module Kafka
 
           messages_for_topics = {}
 
-          message_buffer.each do |topic, partition, messages|
-            message_set = Protocol::MessageSet.new(messages: messages)
-            message_set = @compressor.compress(message_set)
-
+          message_buffer.each do |topic, partition, records|
+            record_batch = Protocol::RecordBatch.new(records: records)
             messages_for_topics[topic] ||= {}
-            messages_for_topics[topic][partition] = message_set
+            messages_for_topics[topic][partition] = record_batch
           end
 
           response = broker.produce(

data/lib/kafka/protocol/decoder.rb

@@ -2,11 +2,12 @@
 
 module Kafka
   module Protocol
-
     # A decoder wraps an IO object, making it easy to read specific data types
     # from it. The Kafka protocol is not self-describing, so a client must call
     # these methods in just the right order for things to work.
     class Decoder
+      VARINT_MASK = 0b10000000
+
       def self.from_string(str)
         new(StringIO.new(str))
       end
@@ -22,6 +23,16 @@ module Kafka
         @io.eof?
       end
 
+      # Get some next bytes without touching the current io offset
+      #
+      # @return [Integer]
+      def peek(offset, length)
+        data = @io.read(offset + length)
+        return [] if data.nil?
+        @io.ungetc(data)
+        data.bytes[offset, offset + length] || []
+      end
+
       # Decodes an 8-bit boolean from the IO object.
       #
       # @return [Boolean]
@@ -70,6 +81,15 @@ module Kafka
         size.times.map(&block)
       end
 
+      # Decodes an array from the IO object.
+      # Just like #array except the size is in varint format
+      #
+      # @return [Array]
+      def varint_array(&block)
+        size = varint
+        size.times.map(&block)
+      end
+
       # Decodes a string from the IO object.
       #
       # @return [String]
@@ -83,6 +103,35 @@ module Kafka
         end
       end
 
+      # Decodes a string from the IO object, the size is in varint format
+      #
+      # @return [String]
+      def varint_string
+        size = varint
+
+        if size == -1
+          nil
+        else
+          read(size)
+        end
+      end
+
+      # Read an integer under varints serializing from the IO object.
+      # https://developers.google.com/protocol-buffers/docs/encoding#varints
+      #
+      # @return [Integer]
+      def varint
+        group = 0
+        data = 0
+        loop do
+          chunk = int8
+          data |= (chunk & (~VARINT_MASK)) << group
+          group += 7
+          break if (chunk & VARINT_MASK) == 0
+        end
+        data & 0b1 != 0 ? ~(data >> 1) : (data >> 1)
+      end
+
       # Decodes a list of bytes from the IO object.
       #
       # @return [String]
@@ -96,6 +145,19 @@ module Kafka
         end
       end
 
+      # Decodes a list of bytes from the IO object. The size is in varint format
+      #
+      # @return [String]
+      def varint_bytes
+        size =  varint
+
+        if size == -1
+          nil
+        else
+          read(size)
+        end
+      end
+
       # Reads the specified number of bytes from the IO object, returning them
       # as a String.
       #

data/lib/kafka/protocol/encoder.rb

@@ -4,10 +4,10 @@ require "stringio"
 
 module Kafka
   module Protocol
-
     # An encoder wraps an IO object, making it easy to write specific data types
     # to it.
     class Encoder
+      VARINT_MASK = 0b10000000
 
       # Initializes a new encoder.
       #
@@ -85,6 +85,20 @@ module Kafka
         end
       end
 
+      # Writes an array to the IO object.
+      # Just like #write_array, unless the size is under varint format
+      #
+      # @param array [Array]
+      # @return [nil]
+      def write_varint_array(array, &block)
+        if array.nil?
+          write_varint(-1)
+        else
+          write_varint(array.size)
+          array.each(&block)
+        end
+      end
+
       # Writes a string to the IO object.
       #
       # @param string [String]
@@ -98,6 +112,40 @@ module Kafka
         end
       end
 
+      # Writes a string to the IO object, the size is under varint format
+      #
+      # @param string [String]
+      # @return [nil]
+      def write_varint_string(string)
+        if string.nil?
+          write_varint(-1)
+        else
+          write_varint(string.bytesize)
+          write(string)
+        end
+      end
+
+      # Writes an integer under varints serializing to the IO object.
+      # https://developers.google.com/protocol-buffers/docs/encoding#varints
+      #
+      # @param string [Integer]
+      # @return [nil]
+      def write_varint(int)
+        int = int << 1
+        int = ~int | 1 if int < 0
+
+        loop do
+          chunk = int & (~VARINT_MASK)
+          int = int >> 7
+          if int == 0
+            write_int8(chunk)
+            return
+          else
+            write_int8(chunk | VARINT_MASK)
+          end
+        end
+      end
+
       # Writes a byte string to the IO object.
       #
       # @param bytes [String]
@@ -111,6 +159,19 @@ module Kafka
         end
       end
 
+      # Writes a byte string to the IO object, the size is under varint format
+      #
+      # @param bytes [String]
+      # @return [nil]
+      def write_varint_bytes(bytes)
+        if bytes.nil?
+          write_varint(-1)
+        else
+          write_varint(bytes.bytesize)
+          write(bytes)
+        end
+      end
+
       # Encodes an object into a new buffer.
       #
       # @param object [#encode] the object that will encode itself.

data/lib/kafka/protocol/fetch_request.rb

@@ -17,6 +17,8 @@ module Kafka
     #       MaxBytes => int32
     #
     class FetchRequest
+      ISOLATION_READ_UNCOMMITTED = 0
+      ISOLATION_READ_COMMITTED = 1
 
       # @param max_wait_time [Integer]
       # @param min_bytes [Integer]
@@ -34,7 +36,7 @@ module Kafka
       end
 
       def api_version
-        3
+        4
       end
 
       def response_class
@@ -46,6 +48,7 @@ module Kafka
         encoder.write_int32(@max_wait_time)
         encoder.write_int32(@min_bytes)
         encoder.write_int32(@max_bytes)
+        encoder.write_int8(ISOLATION_READ_COMMITTED)
 
         encoder.write_array(@topics) do |topic, partitions|
           encoder.write_string(topic)

data/lib/kafka/protocol/fetch_response.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "kafka/protocol/message_set"
+require "kafka/protocol/record_batch"
 
 module Kafka
   module Protocol
@@ -17,15 +18,20 @@ module Kafka
     #       MessageSetSize => int32
     #
     class FetchResponse
+      MAGIC_BYTE_OFFSET = 16
+      MAGIC_BYTE_LENGTH = 1
+
       class FetchedPartition
         attr_reader :partition, :error_code
-        attr_reader :highwater_mark_offset, :messages
+        attr_reader :highwater_mark_offset, :last_stable_offset, :aborted_transactions, :messages
 
-        def initialize(partition:, error_code:, highwater_mark_offset:, messages:)
+        def initialize(partition:, error_code:, highwater_mark_offset:, last_stable_offset:, aborted_transactions:, messages:)
           @partition = partition
           @error_code = error_code
           @highwater_mark_offset = highwater_mark_offset
           @messages = messages
+          @last_stable_offset = last_stable_offset
+          @aborted_transactions = aborted_transactions
         end
       end
 
@@ -55,15 +61,36 @@ module Kafka
             partition = decoder.int32
             error_code = decoder.int16
             highwater_mark_offset = decoder.int64
+            last_stable_offset = decoder.int64
+
+            aborted_transactions = decoder.array do
+              producer_id = decoder.int64
+              first_offset = decoder.int64
+              {
+                producer_id: producer_id,
+                first_offset: first_offset
+              }
+            end
+
+            messages_decoder = Decoder.from_string(decoder.bytes)
+            messages = []
+            magic_byte = messages_decoder.peek(MAGIC_BYTE_OFFSET, MAGIC_BYTE_LENGTH)[0].to_i
 
-            message_set_decoder = Decoder.from_string(decoder.bytes)
-            message_set = MessageSet.decode(message_set_decoder)
+            if magic_byte == RecordBatch::MAGIC_BYTE
+              record_batch = RecordBatch.decode(messages_decoder)
+              messages = record_batch.records
+            else
+              message_set = MessageSet.decode(messages_decoder)
+              messages = message_set.messages
+            end
 
             FetchedPartition.new(
               partition: partition,
               error_code: error_code,
               highwater_mark_offset: highwater_mark_offset,
-              messages: message_set.messages,
+              last_stable_offset: last_stable_offset,
+              aborted_transactions: aborted_transactions,
+              messages: messages
             )
           end
 

data/lib/kafka/protocol/produce_request.rb

@@ -27,15 +27,17 @@ module Kafka
     #         Value => bytes
     #
     class ProduceRequest
-      attr_reader :required_acks, :timeout, :messages_for_topics
+      attr_reader :transactional_id, :required_acks, :timeout, :messages_for_topics, :compressor
 
       # @param required_acks [Integer]
       # @param timeout [Integer]
       # @param messages_for_topics [Hash]
-      def initialize(required_acks:, timeout:, messages_for_topics:)
+      def initialize(transactional_id: nil, required_acks:, timeout:, messages_for_topics:, compressor: nil)
+        @transactional_id = transactional_id
         @required_acks = required_acks
         @timeout = timeout
         @messages_for_topics = messages_for_topics
+        @compressor = compressor
       end
 
       def api_key
@@ -43,7 +45,7 @@ module Kafka
       end
 
       def api_version
-        2
+        3
       end
 
       def response_class
@@ -59,24 +61,32 @@ module Kafka
       end
 
       def encode(encoder)
+        encoder.write_string(@transactional_id)
         encoder.write_int16(@required_acks)
         encoder.write_int32(@timeout)
 
         encoder.write_array(@messages_for_topics) do |topic, messages_for_partition|
           encoder.write_string(topic)
 
-          encoder.write_array(messages_for_partition) do |partition, message_set|
+          encoder.write_array(messages_for_partition) do |partition, record_batch|
             encoder.write_int32(partition)
 
-            # When encoding the message set into the request, the bytesize of the message
-            # set must precede the actual data. Therefore we need to encode the entire
-            # message set into a separate buffer first.
-            encoded_message_set = Encoder.encode_with(message_set)
-
-            encoder.write_bytes(encoded_message_set)
+            record_batch.fulfill_relative_data
+            encoded_record_batch = compress(record_batch)
+            encoder.write_bytes(encoded_record_batch)
           end
         end
       end
+
+      private
+
+      def compress(record_batch)
+        if @compressor.nil?
+          Protocol::Encoder.encode_with(record_batch)
+        else
+          @compressor.compress(record_batch)
+        end
+      end
     end
   end
 end

data/lib/kafka/protocol/record.rb

@@ -0,0 +1,79 @@
+module Kafka
+  module Protocol
+    class Record
+      attr_reader :key, :value, :headers, :attributes, :bytesize
+      attr_accessor :offset_delta, :timestamp_delta, :offset, :create_time, :is_control_record
+
+      def initialize(
+        key: nil,
+        value:,
+        headers: {},
+        attributes: 0,
+        offset_delta: 0,
+        timestamp_delta: 0,
+        create_time: Time.now,
+        is_control_record: false
+      )
+        @key = key
+        @value = value
+        @headers = headers
+        @attributes = attributes
+
+        @offset_delta = offset_delta
+        @timestamp_delta = timestamp_delta
+        @create_time = create_time
+        @is_control_record = is_control_record
+
+        @bytesize = @key.to_s.bytesize + @value.to_s.bytesize
+      end
+
+      def encode(encoder)
+        record_buffer = StringIO.new
+
+        record_encoder = Encoder.new(record_buffer)
+
+        record_encoder.write_int8(@attributes)
+        record_encoder.write_varint(@timestamp_delta)
+        record_encoder.write_varint(@offset_delta)
+
+        record_encoder.write_varint_string(@key)
+        record_encoder.write_varint_bytes(@value)
+
+        record_encoder.write_varint_array(@headers.to_a) do |header_key, header_value|
+          record_encoder.write_varint_string(header_key)
+          record_encoder.write_varint_bytes(header_value)
+        end
+
+        encoder.write_varint_bytes(record_buffer.string)
+      end
+
+      def self.decode(decoder)
+        record_decoder = Decoder.from_string(decoder.varint_bytes)
+
+        attributes = record_decoder.int8
+        timestamp_delta = record_decoder.varint
+        offset_delta = record_decoder.varint
+
+        key = record_decoder.varint_string
+        value = record_decoder.varint_bytes
+
+        headers = {}
+        record_decoder.varint_array do
+          header_key = record_decoder.varint_string
+          header_value = record_decoder.varint_bytes
+
+          headers[header_key] = header_value
+        end
+
+        new(
+          key: key,
+          value: value,
+          headers: headers,
+          attributes: attributes,
+          offset_delta: offset_delta,
+          timestamp_delta: timestamp_delta
+        )
+      end
+    end
+  end
+end

data/lib/kafka/protocol/record_batch.rb

@@ -0,0 +1,202 @@
+require 'digest/crc32'
+require 'kafka/protocol/record'
+
+module Kafka
+  module Protocol
+    class RecordBatch
+      MAGIC_BYTE = 2
+      # The size of metadata before the real record data
+      RECORD_BATCH_OVERHEAD = 49
+      # Masks to extract information from attributes
+      CODEC_ID_MASK = 0b00000111
+      IN_TRANSACTION_MASK = 0b00010000
+      IS_CONTROL_BATCH_MASK = 0b00100000
+
+      attr_reader :records, :first_offset, :first_timestamp, :partition_leader_epoch, :in_transaction, :is_control_batch, :last_offset_delta, :max_timestamp, :producer_id, :producer_epoch, :first_sequence
+
+      attr_accessor :codec_id
+
+      def initialize(
+          records: [],
+          first_offset: 0,
+          first_timestamp: Time.now,
+          partition_leader_epoch: 0,
+          codec_id: 0,
+          in_transaction: false,
+          is_control_batch: false,
+          last_offset_delta: 0,
+          producer_id: -1,
+          producer_epoch: 0,
+          first_sequence: 0,
+          max_timestamp: Time.now
+      )
+        @records = records
+        @first_offset = first_offset
+        @first_timestamp = first_timestamp
+        @codec_id = codec_id
+
+        # Records verification
+        @last_offset_delta = last_offset_delta
+        @max_timestamp = max_timestamp
+
+        # Transaction information
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+
+        @first_sequence = first_sequence
+        @partition_leader_epoch = partition_leader_epoch
+        @in_transaction = in_transaction
+        @is_control_batch = is_control_batch
+
+        mark_control_record
+      end
+
+      def size
+        @records.size
+      end
+
+      def attributes
+        0x0000 | @codec_id |
+          (@in_transaction ? IN_TRANSACTION_MASK : 0x0) |
+          (@is_control_batch ? IS_CONTROL_BATCH_MASK : 0x0)
+      end
+
+      def encode(encoder)
+        encoder.write_int64(@first_offset)
+
+        record_batch_buffer = StringIO.new
+        record_batch_encoder = Encoder.new(record_batch_buffer)
+
+        record_batch_encoder.write_int32(@partition_leader_epoch)
+        record_batch_encoder.write_int8(MAGIC_BYTE)
+
+        body = encode_record_batch_body
+        crc = Digest::CRC32c.checksum(body)
+
+        record_batch_encoder.write_int32(crc)
+        record_batch_encoder.write(body)
+
+        encoder.write_bytes(record_batch_buffer.string)
+      end
+
+      def encode_record_batch_body
+        buffer = StringIO.new
+        encoder = Encoder.new(buffer)
+
+        encoder.write_int16(attributes)
+        encoder.write_int32(@last_offset_delta)
+        encoder.write_int64((@first_timestamp.to_f * 1000).to_i)
+        encoder.write_int64((@max_timestamp.to_f * 1000).to_i)
+
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_int32(@first_sequence)
+
+        encoder.write_int32(@records.length)
+
+        records_array = encode_record_array
+        if compressed?
+          codec = Compression.find_codec_by_id(@codec_id)
+          records_array = codec.compress(records_array)
+        end
+        encoder.write(records_array)
+
+        buffer.string
+      end
+
+      def encode_record_array
+        buffer = StringIO.new
+        encoder = Encoder.new(buffer)
+        @records.each do |record|
+          record.encode(encoder)
+        end
+        buffer.string
+      end
+
+      def compressed?
+        @codec_id != 0
+      end
+
+      def fulfill_relative_data
+        first_record = records.min_by { |record| record.create_time }
+        @first_timestamp = first_record.nil? ? Time.now : first_record.create_time
+
+        last_record = records.max_by { |record| record.create_time }
+        @max_timestamp = last_record.nil? ? Time.now : last_record.create_time
+
+        records.each_with_index do |record, index|
+          record.offset_delta = index
+          record.timestamp_delta = (record.create_time - first_timestamp).to_i
+        end
+        @last_offset_delta = records.length - 1
+      end
+
+      def self.decode(decoder)
+        first_offset = decoder.int64
+
+        record_batch_raw = decoder.bytes
+        record_batch_decoder = Decoder.from_string(record_batch_raw)
+
+        partition_leader_epoch = record_batch_decoder.int32
+        # Currently, the magic byte is used to distingush legacy MessageSet and
+        # RecordBatch. Therefore, we don't care about magic byte here yet.
+        _magic_byte = record_batch_decoder.int8
+        _crc = record_batch_decoder.int32
+
+        attributes = record_batch_decoder.int16
+        codec_id = attributes & CODEC_ID_MASK
+        in_transaction = (attributes & IN_TRANSACTION_MASK) > 0
+        is_control_batch = (attributes & IS_CONTROL_BATCH_MASK) > 0
+
+        last_offset_delta = record_batch_decoder.int32
+        first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
+        max_timestamp = Time.at(record_batch_decoder.int64 / 1000)
+
+        producer_id = record_batch_decoder.int64
+        producer_epoch = record_batch_decoder.int16
+        first_sequence = record_batch_decoder.int32
+
+        records_array_length = record_batch_decoder.int32
+        records_array_raw = record_batch_decoder.read(
+          record_batch_raw.size - RECORD_BATCH_OVERHEAD
+        )
+        if codec_id != 0
+          codec = Compression.find_codec_by_id(codec_id)
+          records_array_raw = codec.decompress(records_array_raw)
+        end
+
+        records_array_decoder = Decoder.from_string(records_array_raw)
+        records_array = []
+        until records_array_decoder.eof?
+          record = Record.decode(records_array_decoder)
+          record.offset = first_offset + record.offset_delta
+          record.create_time = first_timestamp + record.timestamp_delta
+          records_array << record
+        end
+
+        raise InsufficientDataMessage if records_array.length != records_array_length
+
+        new(
+          records: records_array,
+          first_offset: first_offset,
+          first_timestamp: first_timestamp,
+          partition_leader_epoch: partition_leader_epoch,
+          in_transaction: in_transaction,
+          is_control_batch: is_control_batch,
+          last_offset_delta: last_offset_delta,
+          producer_id: producer_id,
+          producer_epoch: producer_epoch,
+          first_sequence: first_sequence,
+          max_timestamp: max_timestamp
+        )
+      end
+
+      def mark_control_record
+        if in_transaction && is_control_batch
+          record = @records.first
+          record.is_control_record = true unless record.nil?
+        end
+      end
+    end
+  end
+end

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.0.alpha1"
+  VERSION = "0.7.0.alpha2"
 end

data/ruby-kafka.gemspec

@@ -27,6 +27,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'digest-crc'
+
   spec.add_development_dependency "bundler", ">= 1.9.5"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.0.alpha1
+  version: 0.7.0.alpha2
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -10,6 +10,20 @@ bindir: exe
 cert_chain: []
 date: 2018-05-24 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: digest-crc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -379,6 +393,8 @@ files:
 - lib/kafka/protocol/offset_fetch_response.rb
 - lib/kafka/protocol/produce_request.rb
 - lib/kafka/protocol/produce_response.rb
+- lib/kafka/protocol/record.rb
+- lib/kafka/protocol/record_batch.rb
 - lib/kafka/protocol/request_message.rb
 - lib/kafka/protocol/sasl_handshake_request.rb
 - lib/kafka/protocol/sasl_handshake_response.rb