ruby-kafka 0.7.6 → 1.1.0

data/lib/kafka/protocol/add_offsets_to_txn_request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddOffsetsToTxnRequest
+      def initialize(transactional_id: nil, producer_id:, producer_epoch:, group_id:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @group_id = group_id
+      end
+
+      def api_key
+        ADD_OFFSETS_TO_TXN_API
+      end
+
+      def response_class
+        AddOffsetsToTxnResponse
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_string(@group_id)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/add_offsets_to_txn_response.rb

@@ -0,0 +1,19 @@
+module Kafka
+  module Protocol
+    class AddOffsetsToTxnResponse
+
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+
+    end
+  end
+end

data/lib/kafka/protocol/join_group_request.rb

@@ -7,13 +7,14 @@ module Kafka
     class JoinGroupRequest
       PROTOCOL_TYPE = "consumer"
 
-      def initialize(group_id:, session_timeout:, member_id:, topics: [])
+      def initialize(group_id:, session_timeout:, rebalance_timeout:, member_id:, topics: [])
         @group_id = group_id
         @session_timeout = session_timeout * 1000 # Kafka wants ms.
+        @rebalance_timeout = rebalance_timeout * 1000 # Kafka wants ms.
         @member_id = member_id || ""
         @protocol_type = PROTOCOL_TYPE
         @group_protocols = {
-          "standard" => ConsumerGroupProtocol.new(topics: ["test-messages"]),
+          "roundrobin" => ConsumerGroupProtocol.new(topics: topics),
         }
       end
 
@@ -21,6 +22,10 @@ module Kafka
         JOIN_GROUP_API
       end
 
+      def api_version
+        1
+      end
+
       def response_class
         JoinGroupResponse
       end
@@ -28,6 +33,7 @@ module Kafka
       def encode(encoder)
         encoder.write_string(@group_id)
         encoder.write_int32(@session_timeout)
+        encoder.write_int32(@rebalance_timeout)
         encoder.write_string(@member_id)
         encoder.write_string(@protocol_type)
 

data/lib/kafka/protocol/metadata_response.rb

@@ -34,7 +34,7 @@ module Kafka
     #
     class MetadataResponse
       class PartitionMetadata
-        attr_reader :partition_id, :leader
+        attr_reader :partition_id, :leader, :replicas
 
         attr_reader :partition_error_code
 

data/lib/kafka/protocol/offset_fetch_request.rb

@@ -12,8 +12,10 @@ module Kafka
         OFFSET_FETCH_API
       end
 
+      # setting topics to nil fetches all offsets for a consumer group
+      # and that feature is only available in API version 2+
       def api_version
-        1
+        @topics.nil? ? 2 : 1
       end
 
       def response_class

data/lib/kafka/protocol/produce_request.rb

@@ -27,6 +27,8 @@ module Kafka
     #         Value => bytes
     #
     class ProduceRequest
+      API_MIN_VERSION = 3
+
       attr_reader :transactional_id, :required_acks, :timeout, :messages_for_topics, :compressor
 
       # @param required_acks [Integer]
@@ -45,7 +47,7 @@ module Kafka
       end
 
       def api_version
-        3
+        compressor.codec.nil? ? API_MIN_VERSION : [compressor.codec.produce_api_min_version, API_MIN_VERSION].max
       end
 
       def response_class

data/lib/kafka/protocol/record_batch.rb

@@ -1,3 +1,4 @@
+require 'bigdecimal'
 require 'digest/crc32'
 require 'kafka/protocol/record'
 
@@ -131,7 +132,7 @@ module Kafka
 
         records.each_with_index do |record, index|
           record.offset_delta = index
-          record.timestamp_delta = (record.create_time - first_timestamp).to_i
+          record.timestamp_delta = ((record.create_time - first_timestamp) * 1000).to_i
         end
         @last_offset_delta = records.length - 1
       end
@@ -167,8 +168,8 @@ module Kafka
         log_append_time = (attributes & TIMESTAMP_TYPE_MASK) != 0
 
         last_offset_delta = record_batch_decoder.int32
-        first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
-        max_timestamp = Time.at(record_batch_decoder.int64 / 1000)
+        first_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
+        max_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
 
         producer_id = record_batch_decoder.int64
         producer_epoch = record_batch_decoder.int16
@@ -188,7 +189,7 @@ module Kafka
         until records_array_decoder.eof?
           record = Record.decode(records_array_decoder)
           record.offset = first_offset + record.offset_delta
-          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta
+          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta / BigDecimal(1000)
           records_array << record
         end
 

data/lib/kafka/protocol/txn_offset_commit_request.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class TxnOffsetCommitRequest
+
+      def api_key
+        TXN_OFFSET_COMMIT_API
+      end
+
+      def api_version
+        2
+      end
+
+      def response_class
+        TxnOffsetCommitResponse
+      end
+
+      def initialize(transactional_id:, group_id:, producer_id:, producer_epoch:, offsets:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @group_id = group_id
+        @offsets = offsets
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_string(@group_id)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+
+        encoder.write_array(@offsets) do |topic, partitions|
+          encoder.write_string(topic)
+          encoder.write_array(partitions) do |partition, offset|
+            encoder.write_int32(partition)
+            encoder.write_int64(offset[:offset])
+            encoder.write_string(nil) # metadata
+            encoder.write_int32(offset[:leader_epoch])
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/kafka/protocol/txn_offset_commit_response.rb

@@ -0,0 +1,18 @@
+module Kafka
+  module Protocol
+    class TxnOffsetCommitResponse
+
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+    end
+  end
+end

data/lib/kafka/sasl/scram.rb

@@ -12,6 +12,7 @@ module Kafka
       }.freeze
 
       def initialize(username:, password:, mechanism: 'sha256', logger:)
+        @semaphore = Mutex.new
         @username = username
         @password = password
         @logger = TaggedLogger.new(logger)
@@ -35,22 +36,24 @@ module Kafka
         @logger.debug "Authenticating #{@username} with SASL #{@mechanism}"
 
         begin
-          msg = first_message
-          @logger.debug "Sending first client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+          @semaphore.synchronize do
+            msg = first_message
+            @logger.debug "Sending first client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          @server_first_message = decoder.bytes
-          @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
+            @server_first_message = decoder.bytes
+            @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
 
-          msg = final_message
-          @logger.debug "Sending final client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+            msg = final_message
+            @logger.debug "Sending final client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          response = parse_response(decoder.bytes)
-          @logger.debug "Received last server SASL SCRAM message: #{response}"
+            response = parse_response(decoder.bytes)
+            @logger.debug "Received last server SASL SCRAM message: #{response}"
 
-          raise FailedScramAuthentication, response['e'] if response['e']
-          raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+            raise FailedScramAuthentication, response['e'] if response['e']
+            raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+          end
         rescue EOFError => e
           raise FailedScramAuthentication, e.message
         end

data/lib/kafka/snappy_codec.rb

@@ -6,6 +6,10 @@ module Kafka
       2
     end
 
+    def produce_api_min_version
+      0
+    end
+
     def load
       require "snappy"
     rescue LoadError

data/lib/kafka/ssl_context.rb

@@ -6,7 +6,7 @@ module Kafka
   module SslContext
     CLIENT_CERT_DELIMITER = "\n-----END CERTIFICATE-----\n"
 
-    def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, client_cert_key_password: nil, client_cert_chain: nil, ca_certs_from_system: nil)
+    def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, client_cert_key_password: nil, client_cert_chain: nil, ca_certs_from_system: nil, verify_hostname: true)
       return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || client_cert_key_password || client_cert_chain || ca_certs_from_system
 
       ssl_context = OpenSSL::SSL::SSLContext.new
@@ -56,6 +56,10 @@ module Kafka
         ssl_context.cert_store = store
       end
 
+      ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      # Verify certificate hostname if supported (ruby >= 2.4.0)
+      ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
+
       ssl_context
     end
   end

data/lib/kafka/ssl_socket_with_timeout.rb

@@ -57,6 +57,7 @@ module Kafka
 
       # once that's connected, we can start initiating the ssl socket
       @ssl_socket = OpenSSL::SSL::SSLSocket.new(@tcp_socket, ssl_context)
+      @ssl_socket.hostname = host
 
       begin
         # Initiate the socket connection in the background. If it doesn't fail

data/lib/kafka/tagged_logger.rb

@@ -1,13 +1,20 @@
-require 'forwardable'
-
 # Basic implementation of a tagged logger that matches the API of
 # ActiveSupport::TaggedLogging.
 
+require 'delegate'
+require 'logger'
+
 module Kafka
-  module TaggedFormatter
+  class TaggedLogger < SimpleDelegator
 
-    def call(severity, timestamp, progname, msg)
-      super(severity, timestamp, progname, "#{tags_text}#{msg}")
+    %i(debug info warn error).each do |method|
+      define_method method do |msg_or_progname, &block|
+        if block_given?
+          super(msg_or_progname, &block)
+        else
+          super("#{tags_text}#{msg_or_progname}")
+        end
+      end
     end
 
     def tagged(*tags)
@@ -44,23 +51,21 @@ module Kafka
       end
     end
 
-  end
-
-  module TaggedLogger
-    extend Forwardable
-    delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
-
-    def self.new(logger)
-      logger ||= Logger.new(nil)
-      return logger if logger.respond_to?(:push_tags) # already included
-      # Ensure we set a default formatter so we aren't extending nil!
-      logger.formatter ||= Logger::Formatter.new
-      logger.formatter.extend TaggedFormatter
-      logger.extend(self)
+    def self.new(logger_or_stream = nil)
+      # don't keep wrapping the same logger over and over again
+      return logger_or_stream if logger_or_stream.is_a?(TaggedLogger)
+      super
     end
 
-    def tagged(*tags)
-      formatter.tagged(*tags) { yield self }
+    def initialize(logger_or_stream = nil)
+      logger = if %w(info debug warn error).all? { |s| logger_or_stream.respond_to?(s) }
+        logger_or_stream
+      elsif logger_or_stream
+        ::Logger.new(logger_or_stream)
+      else
+        ::Logger.new(nil)
+      end
+      super(logger)
     end
 
     def flush

data/lib/kafka/transaction_manager.rb

@@ -217,6 +217,31 @@ module Kafka
       raise
     end
 
+    def send_offsets_to_txn(offsets:, group_id:)
+      force_transactional!
+
+      unless @transaction_state.in_transaction?
+        raise 'Transaction is not valid to send offsets'
+      end
+
+      add_response = transaction_coordinator.add_offsets_to_txn(
+        transactional_id: @transactional_id,
+        producer_id: @producer_id,
+        producer_epoch: @producer_epoch,
+        group_id: group_id
+      )
+      Protocol.handle_error(add_response.error_code)
+
+      send_response = transaction_coordinator.txn_offset_commit(
+        transactional_id: @transactional_id,
+        group_id: group_id,
+        producer_id: @producer_id,
+        producer_epoch: @producer_epoch,
+        offsets: offsets
+      )
+      Protocol.handle_error(send_response.error_code)
+    end
+
     def in_transaction?
       @transaction_state.in_transaction?
     end

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.6"
+  VERSION = "1.1.0"
 end

data/lib/kafka/zstd_codec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kafka
+  class ZstdCodec
+    def codec_id
+      4
+    end
+
+    def produce_api_min_version
+      7
+    end
+
+    def load
+      require "zstd-ruby"
+    rescue LoadError
+      raise LoadError, "using zstd compression requires adding a dependency on the `zstd-ruby` gem to your Gemfile."
+    end
+
+    def compress(data)
+      Zstd.compress(data)
+    end
+
+    def decompress(data)
+      Zstd.decompress(data)
+    end
+  end
+end

data/ruby-kafka.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.homepage      = "https://github.com/zendesk/ruby-kafka"
-  spec.license       = "Apache License Version 2.0"
+  spec.license       = "Apache-2.0"
 
   spec.required_ruby_version = '>= 2.1.0'
 
@@ -36,13 +36,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
-  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "extlz4"
+  spec.add_development_dependency "zstd-ruby"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
+  spec.add_development_dependency "prometheus-client", "~> 0.10.0"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"