ruby-kafka 0.7.8 → 0.7.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f85ddf1d516d86a5cedcf4de17f400cc93e8e20846dd657b3acb85ba6aca1bf6
-  data.tar.gz: d7a55e0fad7767fed575ac2d05b3cba3837402dc97c908e7f47709bd550a484d
+  metadata.gz: 5aa84a39ade791c3bce57152d426e95cb7d29575af3e8a9cb29fcded1c6cdad5
+  data.tar.gz: 46be738fb1a2b71bbbd5d5d3b2fc95b705aef5740f666095eeaf40c9a6980fc4
 SHA512:
-  metadata.gz: 1c4a94639c6c6a889d4be943c4201c60d6d29d872d20fde38b86c54168cb98ea1bfa0b1f151595033841792ee64db73cfdf1138ccd3ad51180a18ad742613230
-  data.tar.gz: 4bde456d53429e192dc2d7dae7a75e009c15d1b793be658fb7402831376556cf9b2ea44eeaf858229a5d2b5e8fa75c2fa74e1ea1f89777a2b5728838e79422d2
+  metadata.gz: 42e8059ab9b49e16a8e9c70e5739c9694d2142d5d440f8e2c18477cc151a24183536ae1548aed66a7e7ee6ea64c96ff2c9aacb63366ed92bd1612af6e52d8300
+  data.tar.gz: 5008c803cc61fbea993f7f2a22ca7ac3ebb448f91cbdca40d540b9f4604dab8b4e38fcf4b20006a16641b5239e78d6045ccbec70168171219b716090ea72ba09

data/.circleci/config.yml

@@ -145,21 +145,53 @@ jobs:
         environment:
           LOG_LEVEL: DEBUG
       - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:2.12-2.1.0
+      - image: wurstmeister/kafka:2.12-2.1.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9092
           KAFKA_PORT: 9092
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
           KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.12-2.1.0
+      - image: wurstmeister/kafka:2.12-2.1.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9093
           KAFKA_PORT: 9093
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
           KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.12-2.1.0
+      - image: wurstmeister/kafka:2.12-2.1.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec --profile --tag functional spec/functional
+
+  kafka-2.2:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.12-2.2.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.2.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.2.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9094
@@ -181,3 +213,4 @@ workflows:
       - kafka-1.1
       - kafka-2.0
       - kafka-2.1
+      - kafka-2.2

data/CHANGELOG.md

@@ -2,7 +2,16 @@
 
 Changes and additions to the library will be listed here.
 
-## Unreleased
+## 0.7.10
+
+- Fix logger again (#762)
+
+## 0.7.9
+
+- Fix SSL authentication for ruby < 2.4.0 (#742)
+- Add metrics for prometheus/client (#739)
+- Do not add nil message entries when ignoring old messages (#746)
+- Scram authentication thread save (#743)
 
 ## 0.7.8
 - Optionally verify hostname on SSL certs (#733)

data/README.md

@@ -108,6 +108,11 @@ Or install it yourself as:
     <td>Limited support</td>
     <td>Limited support</td>
   </tr>
+  <tr>
+    <th>Kafka 2.2</th>
+    <td>Limited support</td>
+    <td>Limited support</td>
+  </tr>
 </table>
 
 This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with the v0.5.x series. There's limited support for Kafka 0.8, and things should work with Kafka 0.11, although there may be performance issues due to changes in the protocol.

data/lib/kafka/fetched_batch_generator.rb

@@ -48,7 +48,7 @@ module Kafka
               partition: @fetched_partition.partition
             )
           end
-        end
+        end.compact
       end
       FetchedBatch.new(
         topic: @topic,

data/lib/kafka/prometheus.rb

@@ -0,0 +1,317 @@
+# frozen_string_literal: true
+
+#
+#  Subscriber to ruby_kafka to report metrics to prometheus
+#
+#  Usage:
+#     require "kafka/prometheus"
+#
+#  Once the file has been required, no further configuration is needed, all operational
+#  metrics are automatically emitted (Unless PROMETHEUS_NO_AUTO_START is set).
+#
+#  By Peter Mustel, T2 Data AB
+#
+begin
+  require 'prometheus/client'
+rescue LoadError
+  warn 'In order to report Kafka client metrics to Prometheus you need to install the `prometheus-client` gem.'
+  raise
+end
+
+require 'active_support/subscriber'
+
+module Kafka
+  module Prometheus
+    SIZE_BUCKETS = [1, 10, 100, 1000, 10_000, 100_000, 1_000_000].freeze
+    LATENCY_BUCKETS = [0.0001, 0.001, 0.01, 0.1, 1.0, 10, 100, 1000].freeze
+    DELAY_BUCKETS = [1, 3, 10, 30, 100, 300, 1000, 3000, 10_000, 30_000].freeze
+
+    class << self
+      attr_accessor :registry
+
+      def start(registry = ::Prometheus::Client.registry)
+        @registry = registry
+        ConnectionSubscriber.attach_to 'connection.kafka'
+        ConsumerSubscriber.attach_to 'consumer.kafka'
+        ProducerSubscriber.attach_to 'producer.kafka'
+        AsyncProducerSubscriber.attach_to 'async_producer.kafka'
+        FetcherSubscriber.attach_to 'fetcher.kafka'
+      end
+    end
+
+    class ConnectionSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @api_calls = Prometheus.registry.counter(:api_calls, 'Total calls')
+        @api_latency = Prometheus.registry.histogram(:api_latency, 'Latency', {}, LATENCY_BUCKETS)
+        @api_request_size = Prometheus.registry.histogram(:api_request_size, 'Request size', {}, SIZE_BUCKETS)
+        @api_response_size = Prometheus.registry.histogram(:api_response_size, 'Response size', {}, SIZE_BUCKETS)
+        @api_errors = Prometheus.registry.counter(:api_errors, 'Errors')
+      end
+
+      def request(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          api: event.payload.fetch(:api, 'unknown'),
+          broker: event.payload.fetch(:broker_host)
+        }
+        request_size = event.payload.fetch(:request_size, 0)
+        response_size = event.payload.fetch(:response_size, 0)
+
+        @api_calls.increment(key)
+        @api_latency.observe(key, event.duration)
+        @api_request_size.observe(key, request_size)
+        @api_response_size.observe(key, response_size)
+        @api_errors.increment(key) if event.payload.key?(:exception)
+      end
+    end
+
+    class ConsumerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @process_messages = Prometheus.registry.counter(:consumer_process_messages, 'Total messages')
+        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, 'Total errors')
+        @process_message_latency =
+          Prometheus.registry.histogram(:consumer_process_message_latency, 'Latency', {}, LATENCY_BUCKETS)
+        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, 'Offset lag')
+        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, 'Time lag of message')
+        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, 'Total errors in batch')
+        @process_batch_latency =
+          Prometheus.registry.histogram(:consumer_process_batch_latency, 'Latency in batch', {}, LATENCY_BUCKETS)
+        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, 'Size of batch', {}, SIZE_BUCKETS)
+        @join_group = Prometheus.registry.histogram(:consumer_join_group, 'Time to join group', {}, DELAY_BUCKETS)
+        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, 'Total error in joining group')
+        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, 'Time to sync group', {}, DELAY_BUCKETS)
+        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, 'Total error in syncing group')
+        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, 'Time to leave group', {}, DELAY_BUCKETS)
+        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, 'Total error in leaving group')
+        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, 'Pause duration')
+      end
+
+      def process_message(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+
+        offset_lag = event.payload.fetch(:offset_lag)
+        create_time = event.payload.fetch(:create_time)
+
+        time_lag = create_time && ((Time.now - create_time) * 1000).to_i
+
+        if event.payload.key?(:exception)
+          @process_message_errors.increment(key)
+        else
+          @process_message_latency.observe(key, event.duration)
+          @process_messages.increment(key)
+        end
+
+        @offset_lag.set(key, offset_lag)
+
+        # Not all messages have timestamps.
+        return unless time_lag
+
+        @time_lag.set(key, time_lag)
+      end
+
+      def process_batch(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+        message_count = event.payload.fetch(:message_count)
+
+        if event.payload.key?(:exception)
+          @process_batch_errors.increment(key)
+        else
+          @process_batch_latency.observe(key, event.duration)
+          @process_messages.increment(key, message_count)
+        end
+      end
+
+      def fetch_batch(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+        offset_lag = event.payload.fetch(:offset_lag)
+        batch_size = event.payload.fetch(:message_count)
+
+        @batch_size.observe(key, batch_size)
+        @offset_lag.set(key, offset_lag)
+      end
+
+      def join_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @join_group.observe(key, event.duration)
+
+        @join_group_errors.increment(key) if event.payload.key?(:exception)
+      end
+
+      def sync_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @sync_group.observe(key, event.duration)
+
+        @sync_group_errors.increment(key) if event.payload.key?(:exception)
+      end
+
+      def leave_group(event)
+        key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
+        @leave_group.observe(key, event.duration)
+
+        @leave_group_errors.increment(key) if event.payload.key?(:exception)
+      end
+
+      def pause_status(event)
+        key = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition)
+        }
+
+        duration = event.payload.fetch(:duration)
+        @pause_duration.set(key, duration)
+      end
+    end
+
+    class ProducerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, 'Produced messages total')
+        @produce_message_size =
+          Prometheus.registry.histogram(:producer_message_size, 'Message size', {}, SIZE_BUCKETS)
+        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, 'Buffer size', {}, SIZE_BUCKETS)
+        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, 'Buffer fill ratio')
+        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, 'Buffer fill percentage')
+        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, 'Produce errors')
+        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, 'Deliver error')
+        @deliver_latency =
+          Prometheus.registry.histogram(:producer_deliver_latency, 'Delivery latency', {}, LATENCY_BUCKETS)
+        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, 'Total count of delivered messages')
+        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, 'Delivery attempts')
+        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, 'Ack')
+        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, 'Ack delay', {}, LATENCY_BUCKETS)
+        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, 'Ack errors')
+      end
+
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        key = { client: client, topic: event.payload.fetch(:topic) }
+
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+        max_buffer_size = event.payload.fetch(:max_buffer_size)
+        buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
+        buffer_fill_percentage = buffer_fill_ratio * 100.0
+
+        # This gets us the write rate.
+        @produce_messages.increment(key)
+        @produce_message_size.observe(key, message_size)
+
+        # This gets us the avg/max buffer size per producer.
+        @buffer_size.observe({ client: client }, buffer_size)
+
+        # This gets us the avg/max buffer fill ratio per producer.
+        @buffer_fill_ratio.observe({ client: client }, buffer_fill_ratio)
+        @buffer_fill_percentage.observe({ client: client }, buffer_fill_percentage)
+      end
+
+      def buffer_overflow(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+        @produce_errors.increment(key)
+      end
+
+      def deliver_messages(event)
+        key = { client: event.payload.fetch(:client_id) }
+        message_count = event.payload.fetch(:delivered_message_count)
+        attempts = event.payload.fetch(:attempts)
+
+        @deliver_errors.increment(key) if event.payload.key?(:exception)
+        @deliver_latency.observe(key, event.duration)
+
+        # Messages delivered to Kafka:
+        @deliver_messages.increment(key, message_count)
+
+        # Number of attempts to deliver messages:
+        @deliver_attempts.observe(key, attempts)
+      end
+
+      def ack_message(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        # Number of messages ACK'd for the topic.
+        @ack_messages.increment(key)
+
+        # Histogram of delay between a message being produced and it being ACK'd.
+        @ack_delay.observe(key, event.payload.fetch(:delay))
+      end
+
+      def topic_error(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        @ack_errors.increment(key)
+      end
+    end
+
+    class AsyncProducerSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, 'Queue size', {}, SIZE_BUCKETS)
+        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, 'Queue fill ratio')
+        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, 'Producer errors')
+        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, 'Dropped messages')
+      end
+
+      def enqueue_message(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+
+        queue_size = event.payload.fetch(:queue_size)
+        max_queue_size = event.payload.fetch(:max_queue_size)
+        queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
+
+        # This gets us the avg/max queue size per producer.
+        @queue_size.observe(key, queue_size)
+
+        # This gets us the avg/max queue fill ratio per producer.
+        @queue_fill_ratio.observe(key, queue_fill_ratio)
+      end
+
+      def buffer_overflow(event)
+        key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
+        @produce_errors.increment(key)
+      end
+
+      def drop_messages(event)
+        key = { client: event.payload.fetch(:client_id) }
+        message_count = event.payload.fetch(:message_count)
+
+        @dropped_messages.increment(key, message_count)
+      end
+    end
+
+    class FetcherSubscriber < ActiveSupport::Subscriber
+      def initialize
+        super
+        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, 'Queue size')
+      end
+
+      def loop(event)
+        queue_size = event.payload.fetch(:queue_size)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        @queue_size.set({ client: client, group_id: group_id }, queue_size)
+      end
+    end
+  end
+end
+
+# To enable testability, it is possible to skip the start until test time
+Kafka::Prometheus.start unless defined?(PROMETHEUS_NO_AUTO_START)

data/lib/kafka/protocol/record_batch.rb

@@ -1,3 +1,4 @@
+require 'bigdecimal'
 require 'digest/crc32'
 require 'kafka/protocol/record'
 
@@ -131,7 +132,7 @@ module Kafka
 
         records.each_with_index do |record, index|
           record.offset_delta = index
-          record.timestamp_delta = (record.create_time - first_timestamp).to_i
+          record.timestamp_delta = ((record.create_time - first_timestamp) * 1000).to_i
         end
         @last_offset_delta = records.length - 1
       end
@@ -167,8 +168,8 @@ module Kafka
         log_append_time = (attributes & TIMESTAMP_TYPE_MASK) != 0
 
         last_offset_delta = record_batch_decoder.int32
-        first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
-        max_timestamp = Time.at(record_batch_decoder.int64 / 1000)
+        first_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
+        max_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
 
         producer_id = record_batch_decoder.int64
         producer_epoch = record_batch_decoder.int16
@@ -188,7 +189,7 @@ module Kafka
         until records_array_decoder.eof?
           record = Record.decode(records_array_decoder)
           record.offset = first_offset + record.offset_delta
-          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta
+          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta / BigDecimal(1000)
           records_array << record
         end
 

data/lib/kafka/sasl/scram.rb

@@ -12,6 +12,7 @@ module Kafka
       }.freeze
 
       def initialize(username:, password:, mechanism: 'sha256', logger:)
+        @semaphore = Mutex.new
         @username = username
         @password = password
         @logger = TaggedLogger.new(logger)
@@ -35,22 +36,24 @@ module Kafka
         @logger.debug "Authenticating #{@username} with SASL #{@mechanism}"
 
         begin
-          msg = first_message
-          @logger.debug "Sending first client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+          @semaphore.synchronize do
+            msg = first_message
+            @logger.debug "Sending first client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          @server_first_message = decoder.bytes
-          @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
+            @server_first_message = decoder.bytes
+            @logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
 
-          msg = final_message
-          @logger.debug "Sending final client SASL SCRAM message: #{msg}"
-          encoder.write_bytes(msg)
+            msg = final_message
+            @logger.debug "Sending final client SASL SCRAM message: #{msg}"
+            encoder.write_bytes(msg)
 
-          response = parse_response(decoder.bytes)
-          @logger.debug "Received last server SASL SCRAM message: #{response}"
+            response = parse_response(decoder.bytes)
+            @logger.debug "Received last server SASL SCRAM message: #{response}"
 
-          raise FailedScramAuthentication, response['e'] if response['e']
-          raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+            raise FailedScramAuthentication, response['e'] if response['e']
+            raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
+          end
         rescue EOFError => e
           raise FailedScramAuthentication, e.message
         end

data/lib/kafka/ssl_context.rb

@@ -55,7 +55,8 @@ module Kafka
         end
         ssl_context.cert_store = store
         ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        ssl_context.verify_hostname = verify_hostname
+        # Verify certificate hostname if supported (ruby >= 2.4.0)
+        ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
       end
 
       ssl_context

data/lib/kafka/tagged_logger.rb

@@ -1,13 +1,19 @@
-require 'forwardable'
-
 # Basic implementation of a tagged logger that matches the API of
 # ActiveSupport::TaggedLogging.
 
+require 'logger'
+
 module Kafka
-  module TaggedFormatter
+  class TaggedLogger < SimpleDelegator
 
-    def call(severity, timestamp, progname, msg)
-      super(severity, timestamp, progname, "#{tags_text}#{msg}")
+    %i(debug info warn error).each do |method|
+      define_method method do |msg_or_progname, &block|
+        if block_given?
+          super(msg_or_progname, &block)
+        else
+          super("#{tags_text}#{msg_or_progname}")
+        end
+      end
     end
 
     def tagged(*tags)
@@ -44,23 +50,21 @@ module Kafka
       end
     end
 
-  end
-
-  module TaggedLogger
-    extend Forwardable
-    delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
-
-    def self.new(logger)
-      logger ||= Logger.new(nil)
-      return logger if logger.respond_to?(:push_tags) # already included
-      # Ensure we set a default formatter so we aren't extending nil!
-      logger.formatter ||= Logger::Formatter.new
-      logger.formatter.extend TaggedFormatter
-      logger.extend(self)
+    def self.new(logger_or_stream = nil)
+      # don't keep wrapping the same logger over and over again
+      return logger_or_stream if logger_or_stream.is_a?(TaggedLogger)
+      super
     end
 
-    def tagged(*tags)
-      formatter.tagged(*tags) { yield self }
+    def initialize(logger_or_stream = nil)
+      logger = if %w(info debug warn error).all? { |s| logger_or_stream.respond_to?(s) }
+        logger_or_stream
+      elsif logger_or_stream
+        ::Logger.new(logger_or_stream)
+      else
+        ::Logger.new(nil)
+      end
+      super(logger)
     end
 
     def flush

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.8"
+  VERSION = "0.7.10"
 end

data/ruby-kafka.gemspec

@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
   spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
+  spec.add_development_dependency "prometheus-client"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.10
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-31 00:00:00.000000000 Z
+date: 2019-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -240,6 +240,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -374,6 +388,7 @@ files:
 - lib/kafka/pending_message_queue.rb
 - lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
+- lib/kafka/prometheus.rb
 - lib/kafka/protocol.rb
 - lib/kafka/protocol/add_offsets_to_txn_request.rb
 - lib/kafka/protocol/add_offsets_to_txn_response.rb