ruby-kafka 0.7.8 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f85ddf1d516d86a5cedcf4de17f400cc93e8e20846dd657b3acb85ba6aca1bf6
-  data.tar.gz: d7a55e0fad7767fed575ac2d05b3cba3837402dc97c908e7f47709bd550a484d
+  metadata.gz: 9facc22cb8739a683d4be6ee8389f0e2c6e7a6530e17e855ee6333aabdf428f0
+  data.tar.gz: d0e979d9e2e0ccf776d7d062e807367ff6380fa4f2863ecf6c414f3a7a3dc1fa
 SHA512:
-  metadata.gz: 1c4a94639c6c6a889d4be943c4201c60d6d29d872d20fde38b86c54168cb98ea1bfa0b1f151595033841792ee64db73cfdf1138ccd3ad51180a18ad742613230
-  data.tar.gz: 4bde456d53429e192dc2d7dae7a75e009c15d1b793be658fb7402831376556cf9b2ea44eeaf858229a5d2b5e8fa75c2fa74e1ea1f89777a2b5728838e79422d2
+  metadata.gz: b339415a8ba234fe78d0fce1c36933f7d39e79561dfd296a1cdb580877f3e5ab7fe5c8119709458b5207f9dc9375638d7a1e93c5c24adc155841dca07b783e5b
+  data.tar.gz: b54e211a11096e98c6460958de7097c4293fdee3dac18e117f75e9ce2c1eeb6b749a8a8f2b5ba8e559ee75c4fa33a8a299c68e37bab9a1f7e16194d8dd8ef92f

data/CHANGELOG.md

@@ -2,7 +2,12 @@
 
 Changes and additions to the library will be listed here.
 
-## Unreleased
+## 0.7.9
+
+- Fix SSL authentication for ruby < 2.4.0 (#742)
+- Add metrics for prometheus/client (#739)
+- Do not add nil message entries when ignoring old messages (#746)
+- Scram authentication thread save (#743)
 
 ## 0.7.8
 - Optionally verify hostname on SSL certs (#733)

data/lib/kafka/fetched_batch_generator.rb

@@ -48,7 +48,7 @@ module Kafka
               partition: @fetched_partition.partition
             )
           end
-        end
+        end.compact
       end
       FetchedBatch.new(
         topic: @topic,

data/lib/kafka/prometheus.rb

@@ -0,0 +1,317 @@
+# frozen_string_literal: true
+
+#
+#  Subscriber to ruby_kafka to report metrics to prometheus
+#
+#  Usage:
+#     require "kafka/prometheus"
+#
+#  Once the file has been required, no further configuration is needed, all operational
+#  metrics are automatically emitted (Unless PROMETHEUS_NO_AUTO_START is set).
+#
+#  By Peter Mustel, T2 Data AB
+#
+begin
+  require 'prometheus/client'
+rescue LoadError
+  warn 'In order to report Kafka client metrics to Prometheus you need to install the `prometheus-client` gem.'
+  raise
+end
+
+require 'active_support/subscriber'
+
+module Kafka
+  module Prometheus
+    SIZE_BUCKETS = [1, 10, 100, 1000, 10_000, 100_000, 1_000_000].freeze
+    LATENCY_BUCKETS = [0.0001, 0.001, 0.01, 0.1, 1.0, 10, 100, 1000].freeze
+    DELAY_BUCKETS = [1, 3, 10, 30, 100, 300, 1000, 3000, 10_000, 30_000].freeze
+
+    class << self
+      attr_accessor :registry
+
+      def start(registry = ::Prometheus::Client.registry)
+        @registry = registry
+        ConnectionSubscriber.attach_to 'connection.kafka'
+        ConsumerSubscriber.attach_to 'consumer.kafka'
+        ProducerSubscriber.attach_to 'producer.kafka'
+        AsyncProducerSubscriber.attach_to 'async_producer.kafka'
+        FetcherSubscriber.attach_to 'fetcher.kafka'
+      end
+    end
+
+    class ConnectionSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @api_calls = Prometheus.registry.counter(:api_calls, 'Total calls')
+        @api_latency = Prometheus.registry.histogram(:api_latency, 'Latency', {}, LATENCY_BUCKETS)
+        @api_request_size = Prometheus.registry.histogram(:api_request_size, 'Request size', {}, SIZE_BUCKETS)
+        @api_response_size = Prometheus.registry.histogram(:api_response_size, 'Response size', {}, SIZE_BUCKETS)
+        @api_errors = Prometheus.registry.counter(:api_errors, 'Errors')
+      end
+
+      def request(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          api: event.payload.fetch(:api, 'unknown'),
+          broker: event.payload.fetch(:broker_host)
+        }
+        request_size = event.payload.fetch(:request_size, 0)
+        response_size = event.payload.fetch(:response_size, 0)
+
+        @api_calls.increment(key)
+        @api_latency.observe(key, event.duration)
+        @api_request_size.observe(key, request_size)
+        @api_response_size.observe(key, response_size)
+        @api_errors.increment(key) if event.payload.key?(:exception)
+      end
+    end
+
+    class ConsumerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @process_messages = Prometheus.registry.counter(:consumer_process_messages, 'Total messages')
+        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, 'Total errors')
+        @process_message_latency =
+          Prometheus.registry.histogram(:consumer_process_message_latency, 'Latency', {}, LATENCY_BUCKETS)
+        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, 'Offset lag')
+        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, 'Time lag of message')
+        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, 'Total errors in batch')
+        @process_batch_latency =
+          Prometheus.registry.histogram(:consumer_process_batch_latency, 'Latency in batch', {}, LATENCY_BUCKETS)
+        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, 'Size of batch', {}, SIZE_BUCKETS)
+        @join_group = Prometheus.registry.histogram(:consumer_join_group, 'Time to join group', {}, DELAY_BUCKETS)
+        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, 'Total error in joining group')
+        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, 'Time to sync group', {}, DELAY_BUCKETS)
+        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, 'Total error in syncing group')
+        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, 'Time to leave group', {}, DELAY_BUCKETS)
+        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, 'Total error in leaving group')
+        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, 'Pause duration')
+      end
+
+      def process_message(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+
+        offset_lag = event.payload.fetch(:offset_lag)
+        create_time = event.payload.fetch(:create_time)
+
+        time_lag = create_time && ((Time.now - create_time) * 1000).to_i
+
+        if event.payload.key?(:exception)
+          @process_message_errors.increment(key)
+        else
+          @process_message_latency.observe(key, event.duration)
+          @process_messages.increment(key)
+        end
+
+        @offset_lag.set(key, offset_lag)
+
+        # Not all messages have timestamps.
+        return unless time_lag
+
+        @time_lag.set(key, time_lag)
+      end
+
+      def process_batch(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+        message_count = event.payload.fetch(:message_count)
+
+        if event.payload.key?(:exception)
+          @process_batch_errors.increment(key)
+        else
+          @process_batch_latency.observe(key, event.duration)
+          @process_messages.increment(key, message_count)
+        end
+      end
+
+      def fetch_batch(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+        offset_lag = event.payload.fetch(:offset_lag)
+        batch_size = event.payload.fetch(:message_count)
+
+        @batch_size.observe(key, batch_size)
+        @offset_lag.set(key, offset_lag)
+      end
+
+      def join_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @join_group.observe(key, event.duration)
+
+        @join_group_errors.increment(key) if event.payload.key?(:exception)
+      end
+
+      def sync_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @sync_group.observe(key, event.duration)
+
+        @sync_group_errors.increment(key) if event.payload.key?(:exception)
+      end
+
+      def leave_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @leave_group.observe(key, event.duration)
+
+        @leave_group_errors.increment(key) if event.payload.key?(:exception)
+      end
+
+      def pause_status(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+
+        duration = event.payload.fetch(:duration)
+        @pause_duration.set(key, duration)
+      end
+    end
+
+    class ProducerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, 'Produced messages total')
+        @produce_message_size =
+          Prometheus.registry.histogram(:producer_message_size, 'Message size', {}, SIZE_BUCKETS)
+        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, 'Buffer size', {}, SIZE_BUCKETS)
+        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, 'Buffer fill ratio')
+        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, 'Buffer fill percentage')
+        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, 'Produce errors')
+        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, 'Deliver error')
+        @deliver_latency =
+          Prometheus.registry.histogram(:producer_deliver_latency, 'Delivery latency', {}, LATENCY_BUCKETS)
+        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, 'Total count of delivered messages')
+        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, 'Delivery attempts')
+        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, 'Ack')
+        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, 'Ack delay', {}, LATENCY_BUCKETS)
+        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, 'Ack errors')
+      end
+
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        key = { client: client, topic: event.payload.fetch(:topic) }
+
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+        max_buffer_size = event.payload.fetch(:max_buffer_size)
+        buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
+        buffer_fill_percentage = buffer_fill_ratio * 100.0
+
+        # This gets us the write rate.
+        @produce_messages.increment(key)
+        @produce_message_size.observe(key, message_size)
+
+        # This gets us the avg/max buffer size per producer.
+        @buffer_size.observe({ client: client }, buffer_size)
+
+        # This gets us the avg/max buffer fill ratio per producer.
+        @buffer_fill_ratio.observe({ client: client }, buffer_fill_ratio)
+        @buffer_fill_percentage.observe({ client: client }, buffer_fill_percentage)
+      end
+
+      def buffer_overflow(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+        @produce_errors.increment(key)
+      end
+
+      def deliver_messages(event)
+        key = { client: event.payload.fetch(:client_id) }
+        message_count = event.payload.fetch(:delivered_message_count)
+        attempts = event.payload.fetch(:attempts)
+
+        @deliver_errors.increment(key) if event.payload.key?(:exception)
+        @deliver_latency.observe(key, event.duration)
+
+        # Messages delivered to Kafka:
+        @deliver_messages.increment(key, message_count)
+
+        # Number of attempts to deliver messages:
+        @deliver_attempts.observe(key, attempts)
+      end
+
+      def ack_message(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        # Number of messages ACK'd for the topic.
+        @ack_messages.increment(key)
+
+        # Histogram of delay between a message being produced and it being ACK'd.
+        @ack_delay.observe(key, event.payload.fetch(:delay))
+      end
+
+      def topic_error(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        @ack_errors.increment(key)
+      end
+    end
+
+    class AsyncProducerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, 'Queue size', {}, SIZE_BUCKETS)
+        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, 'Queue fill ratio')
+        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, 'Producer errors')
+        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, 'Dropped messages')
+      end
+
+      def enqueue_message(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        queue_size = event.payload.fetch(:queue_size)
+        max_queue_size = event.payload.fetch(:max_queue_size)
+        queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
+
+        # This gets us the avg/max queue size per producer.
+        @queue_size.observe(key, queue_size)
+
+        # This gets us the avg/max queue fill ratio per producer.
+        @queue_fill_ratio.observe(key, queue_fill_ratio)
+      end
+
+      def buffer_overflow(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+        @produce_errors.increment(key)
+      end
+
+      def drop_messages(event)
+        key = { client: event.payload.fetch(:client_id) }
+        message_count = event.payload.fetch(:message_count)
+
+        @dropped_messages.increment(key, message_count)
+      end
+    end
+
+    class FetcherSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, 'Queue size')
+      end
+
+      def loop(event)
+        queue_size = event.payload.fetch(:queue_size)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        @queue_size.set({ client: client, group_id: group_id }, queue_size)
+      end
+    end
+  end
+end
+
+# To enable testability, it is possible to skip the start until test time
+Kafka::Prometheus.start unless defined?(PROMETHEUS_NO_AUTO_START)

data/lib/kafka/sasl/scram.rb

@@ -12,6 +12,7 @@ module Kafka
       }.freeze
 
       def initialize(username:, password:, mechanism: 'sha256', logger:)
+        @semaphore = Mutex.new
         @username = username
         @password = password
         @logger = TaggedLogger.new(logger)
@@ -35,22 +36,24 @@ module Kafka
         @logger.debug "Authenticating #{@username} with SASL #{@mechanism}"
 
         begin
-          msg = first_message
-          @logger.debug "Sending first client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+          @semaphore.synchronize do
+            msg = first_message
+            @logger.debug "Sending first client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          @server_first_message = decoder.bytes
-          @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
+            @server_first_message = decoder.bytes
+            @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
 
-          msg = final_message
-          @logger.debug "Sending final client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+            msg = final_message
+            @logger.debug "Sending final client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          response = parse_response(decoder.bytes)
-          @logger.debug "Received last server SASL SCRAM message: #{response}"
+            response = parse_response(decoder.bytes)
+            @logger.debug "Received last server SASL SCRAM message: #{response}"
 
-          raise FailedScramAuthentication, response['e'] if response['e']
-          raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+            raise FailedScramAuthentication, response['e'] if response['e']
+            raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+          end
         rescue EOFError => e
           raise FailedScramAuthentication, e.message
         end

data/lib/kafka/ssl_context.rb

@@ -55,7 +55,8 @@ module Kafka
         end
         ssl_context.cert_store = store
         ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        ssl_context.verify_hostname = verify_hostname
+        # Verify certificate hostname if supported (ruby >= 2.4.0)
+        ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
       end
 
       ssl_context

data/lib/kafka/tagged_logger.rb

@@ -1,13 +1,19 @@
-require 'forwardable'
-
 # Basic implementation of a tagged logger that matches the API of
 # ActiveSupport::TaggedLogging.
 
+require 'logger'
+
 module Kafka
-  module TaggedFormatter
+  class TaggedLogger < SimpleDelegator
 
-    def call(severity, timestamp, progname, msg)
-      super(severity, timestamp, progname, "#{tags_text}#{msg}")
+    %i(debug info warn error fatal).each do |method|
+      define_method method do |msg_or_progname, &block|
+        if block_given?
+          super(msg_or_progname, &block)
+        else
+          super("#{tags_text}#{msg_or_progname}")
+        end
+      end
     end
 
     def tagged(*tags)
@@ -44,23 +50,21 @@ module Kafka
       end
     end
 
-  end
-
-  module TaggedLogger
-    extend Forwardable
-    delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
-
-    def self.new(logger)
-      logger ||= Logger.new(nil)
-      return logger if logger.respond_to?(:push_tags) # already included
-      # Ensure we set a default formatter so we aren't extending nil!
-      logger.formatter ||= Logger::Formatter.new
-      logger.formatter.extend TaggedFormatter
-      logger.extend(self)
+    def self.new(logger_or_stream = nil)
+      # don't keep wrapping the same logger over and over again
+      return logger_or_stream if logger_or_stream.is_a?(TaggedLogger)
+      super
     end
 
-    def tagged(*tags)
-      formatter.tagged(*tags) { yield self }
+    def initialize(logger_or_stream = nil)
+      logger = if logger_or_stream.is_a?(::Logger)
+        logger_or_stream
+      elsif logger_or_stream
+        ::Logger.new(logger_or_stream)
+      else
+        ::Logger.new(nil)
+      end
+      super(logger)
     end
 
     def flush

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.8"
+  VERSION = "0.7.9"
 end

data/ruby-kafka.gemspec

@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
   spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
+  spec.add_development_dependency "prometheus-client"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.9
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-31 00:00:00.000000000 Z
+date: 2019-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -240,6 +240,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -374,6 +388,7 @@ files:
 - lib/kafka/pending_message_queue.rb
 - lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
+- lib/kafka/prometheus.rb
 - lib/kafka/protocol.rb
 - lib/kafka/protocol/add_offsets_to_txn_request.rb
 - lib/kafka/protocol/add_offsets_to_txn_response.rb