ruby-kafka 0.7.8 → 1.1.0

data/lib/kafka/protocol/join_group_request.rb

@@ -7,13 +7,14 @@ module Kafka
     class JoinGroupRequest
       PROTOCOL_TYPE = "consumer"
 
-      def initialize(group_id:, session_timeout:, member_id:, topics: [])
+      def initialize(group_id:, session_timeout:, rebalance_timeout:, member_id:, topics: [])
         @group_id = group_id
         @session_timeout = session_timeout * 1000 # Kafka wants ms.
+        @rebalance_timeout = rebalance_timeout * 1000 # Kafka wants ms.
         @member_id = member_id || ""
         @protocol_type = PROTOCOL_TYPE
         @group_protocols = {
-          "standard" => ConsumerGroupProtocol.new(topics: ["test-messages"]),
+          "roundrobin" => ConsumerGroupProtocol.new(topics: topics),
         }
       end
 
@@ -21,6 +22,10 @@ module Kafka
         JOIN_GROUP_API
       end
 
+      def api_version
+        1
+      end
+
       def response_class
         JoinGroupResponse
       end
@@ -28,6 +33,7 @@ module Kafka
       def encode(encoder)
         encoder.write_string(@group_id)
         encoder.write_int32(@session_timeout)
+        encoder.write_int32(@rebalance_timeout)
         encoder.write_string(@member_id)
         encoder.write_string(@protocol_type)
 

data/lib/kafka/protocol/metadata_response.rb

@@ -34,7 +34,7 @@ module Kafka
     #
     class MetadataResponse
       class PartitionMetadata
-        attr_reader :partition_id, :leader
+        attr_reader :partition_id, :leader, :replicas
 
         attr_reader :partition_error_code
 

data/lib/kafka/protocol/offset_fetch_request.rb

@@ -12,8 +12,10 @@ module Kafka
         OFFSET_FETCH_API
       end
 
+      # setting topics to nil fetches all offsets for a consumer group
+      # and that feature is only available in API version 2+
       def api_version
-        1
+        @topics.nil? ? 2 : 1
       end
 
       def response_class

data/lib/kafka/protocol/record_batch.rb

@@ -1,3 +1,4 @@
+require 'bigdecimal'
 require 'digest/crc32'
 require 'kafka/protocol/record'
 
@@ -131,7 +132,7 @@ module Kafka
 
         records.each_with_index do |record, index|
           record.offset_delta = index
-          record.timestamp_delta = (record.create_time - first_timestamp).to_i
+          record.timestamp_delta = ((record.create_time - first_timestamp) * 1000).to_i
         end
         @last_offset_delta = records.length - 1
       end
@@ -167,8 +168,8 @@ module Kafka
         log_append_time = (attributes & TIMESTAMP_TYPE_MASK) != 0
 
         last_offset_delta = record_batch_decoder.int32
-        first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
-        max_timestamp = Time.at(record_batch_decoder.int64 / 1000)
+        first_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
+        max_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
 
         producer_id = record_batch_decoder.int64
         producer_epoch = record_batch_decoder.int16
@@ -188,7 +189,7 @@ module Kafka
         until records_array_decoder.eof?
           record = Record.decode(records_array_decoder)
           record.offset = first_offset + record.offset_delta
-          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta
+          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta / BigDecimal(1000)
           records_array << record
         end
 

data/lib/kafka/sasl/scram.rb

@@ -12,6 +12,7 @@ module Kafka
       }.freeze
 
       def initialize(username:, password:, mechanism: 'sha256', logger:)
+        @semaphore = Mutex.new
         @username = username
         @password = password
         @logger = TaggedLogger.new(logger)
@@ -35,22 +36,24 @@ module Kafka
         @logger.debug "Authenticating #{@username} with SASL #{@mechanism}"
 
         begin
-          msg = first_message
-          @logger.debug "Sending first client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+          @semaphore.synchronize do
+            msg = first_message
+            @logger.debug "Sending first client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          @server_first_message = decoder.bytes
-          @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
+            @server_first_message = decoder.bytes
+            @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
 
-          msg = final_message
-          @logger.debug "Sending final client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+            msg = final_message
+            @logger.debug "Sending final client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          response = parse_response(decoder.bytes)
-          @logger.debug "Received last server SASL SCRAM message: #{response}"
+            response = parse_response(decoder.bytes)
+            @logger.debug "Received last server SASL SCRAM message: #{response}"
 
-          raise FailedScramAuthentication, response['e'] if response['e']
-          raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+            raise FailedScramAuthentication, response['e'] if response['e']
+            raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+          end
         rescue EOFError => e
           raise FailedScramAuthentication, e.message
         end

data/lib/kafka/ssl_context.rb

@@ -54,10 +54,12 @@ module Kafka
           store.set_default_paths
         end
         ssl_context.cert_store = store
-        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        ssl_context.verify_hostname = verify_hostname
       end
 
+      ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      # Verify certificate hostname if supported (ruby >= 2.4.0)
+      ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
+
       ssl_context
     end
   end

data/lib/kafka/tagged_logger.rb

@@ -1,13 +1,20 @@
-require 'forwardable'
-
 # Basic implementation of a tagged logger that matches the API of
 # ActiveSupport::TaggedLogging.
 
+require 'delegate'
+require 'logger'
+
 module Kafka
-  module TaggedFormatter
+  class TaggedLogger < SimpleDelegator
 
-    def call(severity, timestamp, progname, msg)
-      super(severity, timestamp, progname, "#{tags_text}#{msg}")
+    %i(debug info warn error).each do |method|
+      define_method method do |msg_or_progname, &block|
+        if block_given?
+          super(msg_or_progname, &block)
+        else
+          super("#{tags_text}#{msg_or_progname}")
+        end
+      end
     end
 
     def tagged(*tags)
@@ -44,23 +51,21 @@ module Kafka
       end
     end
 
-  end
-
-  module TaggedLogger
-    extend Forwardable
-    delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
-
-    def self.new(logger)
-      logger ||= Logger.new(nil)
-      return logger if logger.respond_to?(:push_tags) # already included
-      # Ensure we set a default formatter so we aren't extending nil!
-      logger.formatter ||= Logger::Formatter.new
-      logger.formatter.extend TaggedFormatter
-      logger.extend(self)
+    def self.new(logger_or_stream = nil)
+      # don't keep wrapping the same logger over and over again
+      return logger_or_stream if logger_or_stream.is_a?(TaggedLogger)
+      super
     end
 
-    def tagged(*tags)
-      formatter.tagged(*tags) { yield self }
+    def initialize(logger_or_stream = nil)
+      logger = if %w(info debug warn error).all? { |s| logger_or_stream.respond_to?(s) }
+        logger_or_stream
+      elsif logger_or_stream
+        ::Logger.new(logger_or_stream)
+      else
+        ::Logger.new(nil)
+      end
+      super(logger)
     end
 
     def flush

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.8"
+  VERSION = "1.1.0"
 end

data/ruby-kafka.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.homepage      = "https://github.com/zendesk/ruby-kafka"
-  spec.license       = "Apache License Version 2.0"
+  spec.license       = "Apache-2.0"
 
   spec.required_ruby_version = '>= 2.1.0'
 
@@ -36,14 +36,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
-  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "extlz4"
   spec.add_development_dependency "zstd-ruby"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
+  spec.add_development_dependency "prometheus-client", "~> 0.10.0"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 1.1.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-31 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -128,14 +128,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: snappy
   requirement: !ruby/object:Gem::Requirement
@@ -212,7 +218,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -222,7 +228,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -240,6 +246,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -318,6 +338,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".readygo"
 - ".rspec"
@@ -374,6 +395,7 @@ files:
 - lib/kafka/pending_message_queue.rb
 - lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
+- lib/kafka/prometheus.rb
 - lib/kafka/protocol.rb
 - lib/kafka/protocol/add_offsets_to_txn_request.rb
 - lib/kafka/protocol/add_offsets_to_txn_response.rb
@@ -454,7 +476,7 @@ files:
 - ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
-- Apache License Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []