ruby-kafka 0.7.5 → 0.7.6.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af054c6ab2097ed1069b4875fab57e350a562467551a33f8ff46e6aeb2319dc3
-  data.tar.gz: cb72f7416f0a587bf506465c8c52144d82ae0b98c698250b87f0c178b2768cfd
+  metadata.gz: e372f99c56ceb1e59bf7bc9d6cdc6d002637dc41c40b7c3f1d782aeea1e29c40
+  data.tar.gz: 70ec79b24ffb7105de1d01ebc2ee43da6d13778b2813feacbc30707da072d1c0
 SHA512:
-  metadata.gz: 06ad099880c38bed87155117469f55cd2682d9b1e0fae9abc5b44aafa74acf55984db73a03861bb94e08585744139928d6859fb2a9ea93f21dcab6aa1f0ae9cb
-  data.tar.gz: '087d565a6ae886f053323a0c1f2222eaf63f044664544c614d30e0914f145d83a0a58369f9ee343828224b13fa76f54def52fe8cfd94e9773d4089bea62ad8e8'
+  metadata.gz: c072eb8640de239ca77e3397933ed9d6575f42621aacff709d966b4213b0dbef8b6dd198c3d12298167cd07349c8d4438e3127677803f2630c2c405c3e9771eb
+  data.tar.gz: 6b265b87a57d488d36c8b91c223ebb781a6b2c602fae101023b412736bbf9a7ccda1456d66eab107c22409fb1a3e6f7a0552dd274f595ef243b227206350a445

data/.circleci/config.yml

@@ -113,21 +113,53 @@ jobs:
         environment:
           LOG_LEVEL: DEBUG
       - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:2.11-2.0.0
+      - image: wurstmeister/kafka:2.11-2.0.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9092
           KAFKA_PORT: 9092
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
           KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.11-2.0.0
+      - image: wurstmeister/kafka:2.11-2.0.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9093
           KAFKA_PORT: 9093
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
           KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.11-2.0.0
+      - image: wurstmeister/kafka:2.11-2.0.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec --profile --tag functional spec/functional
+
+  kafka-2.1:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.12-2.1.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.1.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.1.0
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9094
@@ -148,3 +180,4 @@ workflows:
       - kafka-1.0.0
       - kafka-1.1
       - kafka-2.0
+      - kafka-2.1

data/CHANGELOG.md

@@ -4,6 +4,11 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## 0.7.6
+- Introduce regex matching in `Consumer#subscribe` (#700)
+- Only rejoin group on error if we're not in shutdown mode (#711)
+- Use `maxTimestamp` for `logAppendTime` timestamps (#706)
+
 ## 0.7.5
 - Distribute partitions across consumer groups when there are few partitions per topic (#681)
 - Fix an issue where a consumer would fail to fetch any messages (#689)

data/README.md

@@ -98,6 +98,16 @@ Or install it yourself as:
     <td>Limited support</td>
     <td>Limited support</td>
   </tr>
+  <tr>
+    <th>Kafka 2.0</th>
+    <td>Limited support</td>
+    <td>Limited support</td>
+  </tr>
+  <tr>
+    <th>Kafka 2.1</th>
+    <td>Limited support</td>
+    <td>Limited support</td>
+  </tr>
 </table>
 
 This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with the v0.5.x series. There's limited support for Kafka 0.8, and things should work with Kafka 0.11, although there may be performance issues due to changes in the protocol.
@@ -107,6 +117,8 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
 - **Kafka 0.10:** Full support for the Producer and Consumer API in ruby-kafka v0.5.x. Note that you _must_ run version 0.10.1 or higher of Kafka due to limitations in 0.10.0.
 - **Kafka 0.11:** Full support for Producer API, limited support for Consumer API in ruby-kafka v0.7.x. New features in 0.11.x includes new Record Batch format, idempotent and transactional production. The missing feature is dirty reading of Consumer API.
 - **Kafka 1.0:** Everything that works with Kafka 0.11 should still work, but so far no features specific to Kafka 1.0 have been added.
+- **Kafka 2.0:** Everything that works with Kafka 1.0 should still work, but so far no features specific to Kafka 2.0 have been added.
+- **Kafka 2.1:** Everything that works with Kafka 2.0 should still work, but so far no features specific to Kafka 2.1 have been added.
 
 This library requires Ruby 2.1 or higher.
 

data/lib/kafka/async_producer.rb

@@ -80,7 +80,7 @@ module Kafka
       @queue = Queue.new
       @max_queue_size = max_queue_size
       @instrumenter = instrumenter
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
 
       @worker = Worker.new(
         queue: @queue,
@@ -189,10 +189,11 @@ module Kafka
         @producer = producer
         @delivery_threshold = delivery_threshold
         @instrumenter = instrumenter
-        @logger = logger
+        @logger = TaggedLogger.new(logger)
       end
 
       def run
+        @logger.push_tags(@producer.to_s)
         @logger.info "Starting async producer in the background..."
 
         loop do
@@ -233,6 +234,7 @@ module Kafka
         @logger.error "Async producer crashed!"
       ensure
         @producer.shutdown
+        @logger.pop_tags
       end
 
       private

data/lib/kafka/broker.rb

@@ -12,7 +12,7 @@ module Kafka
       @host = host
       @port = port
       @node_id = node_id
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
     end
 
     def address_match?(host, port)

data/lib/kafka/broker_pool.rb

@@ -5,7 +5,7 @@ require "kafka/broker"
 module Kafka
   class BrokerPool
     def initialize(connection_builder:, logger:)
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @connection_builder = connection_builder
       @brokers = {}
     end

data/lib/kafka/client.rb

@@ -14,6 +14,7 @@ require "kafka/fetch_operation"
 require "kafka/connection_builder"
 require "kafka/instrumenter"
 require "kafka/sasl_authenticator"
+require "kafka/tagged_logger"
 
 module Kafka
   class Client
@@ -68,7 +69,7 @@ module Kafka
                    sasl_gssapi_keytab: nil, sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
                    sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil,
                    sasl_over_ssl: true, ssl_ca_certs_from_system: false)
-      @logger = logger || Logger.new(nil)
+      @logger = TaggedLogger.new(logger)
       @instrumenter = Instrumenter.new(client_id: client_id)
       @seed_brokers = normalize_seed_brokers(seed_brokers)
 

data/lib/kafka/cluster.rb

@@ -23,7 +23,7 @@ module Kafka
         raise ArgumentError, "At least one seed broker must be configured"
       end
 
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @seed_brokers = seed_brokers
       @broker_pool = broker_pool
       @cluster_info = nil

data/lib/kafka/connection.rb

@@ -52,7 +52,7 @@ module Kafka
     # @return [Connection] a new connection.
     def initialize(host:, port:, client_id:, logger:, instrumenter:, connect_timeout: nil, socket_timeout: nil, ssl_context: nil)
       @host, @port, @client_id = host, port, client_id
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
 
       @connect_timeout = connect_timeout || CONNECT_TIMEOUT
@@ -93,6 +93,7 @@ module Kafka
 
       raise IdleConnection if idle?
 
+      @logger.push_tags(api_name)
       @instrumenter.instrument("request.connection", notification) do
         open unless open?
 
@@ -113,6 +114,8 @@ module Kafka
       close
 
       raise ConnectionError, "Connection error #{e.class}: #{e}"
+    ensure
+      @logger.pop_tags
     end
 
     private

data/lib/kafka/connection_builder.rb

@@ -4,7 +4,7 @@ module Kafka
   class ConnectionBuilder
     def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:, sasl_authenticator:)
       @client_id = client_id
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
       @connect_timeout = connect_timeout
       @socket_timeout = socket_timeout

data/lib/kafka/consumer.rb

@@ -46,7 +46,7 @@ module Kafka
 
     def initialize(cluster:, logger:, instrumenter:, group:, fetcher:, offset_manager:, session_timeout:, heartbeat:)
       @cluster = cluster
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
       @group = group
       @offset_manager = offset_manager
@@ -82,7 +82,8 @@ module Kafka
     # messages to be written. In the former case, set `start_from_beginning`
     # to true (the default); in the latter, set it to false.
     #
-    # @param topic [String] the name of the topic to subscribe to.
+    # @param topic_or_regex [String, Regexp] subscribe to single topic with a string
+    #   or multiple topics matching a regex.
     # @param default_offset [Symbol] whether to start from the beginning or the
     #   end of the topic's partitions. Deprecated.
     # @param start_from_beginning [Boolean] whether to start from the beginning
@@ -93,12 +94,16 @@ module Kafka
     # @param max_bytes_per_partition [Integer] the maximum amount of data fetched
     #   from a single partition at a time.
     # @return [nil]
-    def subscribe(topic, default_offset: nil, start_from_beginning: true, max_bytes_per_partition: 1048576)
+    def subscribe(topic_or_regex, default_offset: nil, start_from_beginning: true, max_bytes_per_partition: 1048576)
       default_offset ||= start_from_beginning ? :earliest : :latest
 
-      @group.subscribe(topic)
-      @offset_manager.set_default_offset(topic, default_offset)
-      @fetcher.subscribe(topic, max_bytes_per_partition: max_bytes_per_partition)
+      if topic_or_regex.is_a?(Regexp)
+        cluster_topics.select { |topic| topic =~ topic_or_regex }.each do |topic|
+          subscribe_to_topic(topic, default_offset, start_from_beginning, max_bytes_per_partition)
+        end
+      else
+        subscribe_to_topic(topic_or_regex, default_offset, start_from_beginning, max_bytes_per_partition)
+      end
 
       nil
     end
@@ -241,7 +246,7 @@ module Kafka
 
             trigger_heartbeat
 
-            return if !@running
+            return if shutting_down?
           end
 
           # We've successfully processed a batch from the partition, so we can clear
@@ -336,7 +341,7 @@ module Kafka
 
           trigger_heartbeat
 
-          return if !@running
+          return if shutting_down?
         end
 
         # We may not have received any messages, but it's still a good idea to
@@ -386,22 +391,23 @@ module Kafka
 
     def consumer_loop
       @running = true
+      @logger.push_tags(@group.to_s)
 
       @fetcher.start
 
-      while @running
+      while running?
         begin
           @instrumenter.instrument("loop.consumer") do
             yield
           end
         rescue HeartbeatError
           make_final_offsets_commit!
-          join_group
+          join_group if running?
         rescue OffsetCommitError
-          join_group
+          join_group if running?
         rescue RebalanceInProgress
           @logger.warn "Group rebalance in progress, re-joining..."
-          join_group
+          join_group if running?
         rescue FetchError, NotLeaderForPartition, UnknownTopicOrPartition
           @cluster.mark_as_stale!
         rescue LeaderNotAvailable => e
@@ -424,6 +430,7 @@ module Kafka
       make_final_offsets_commit!
       @group.leave rescue nil
       @running = false
+      @logger.pop_tags
     end
 
     def make_final_offsets_commit!(attempts = 3)
@@ -505,7 +512,7 @@ module Kafka
 
     def fetch_batches
       # Return early if the consumer has been stopped.
-      return [] if !@running
+      return [] if shutting_down?
 
       join_group unless @group.member?
 
@@ -545,6 +552,14 @@ module Kafka
       @pauses[topic][partition]
     end
 
+    def running?
+      @running
+    end
+
+    def shutting_down?
+      !running?
+    end
+
     def clear_current_offsets(excluding: {})
       @current_offsets.each do |topic, partitions|
         partitions.keep_if do |partition, _|
@@ -552,5 +567,23 @@ module Kafka
         end
       end
     end
+
+    def subscribe_to_topic(topic, default_offset, start_from_beginning, max_bytes_per_partition)
+      @group.subscribe(topic)
+      @offset_manager.set_default_offset(topic, default_offset)
+      @fetcher.subscribe(topic, max_bytes_per_partition: max_bytes_per_partition)
+    end
+
+    def cluster_topics
+      attempts = 0
+      begin
+        attempts += 1
+        @cluster.list_topics
+      rescue Kafka::ConnectionError
+        @cluster.mark_as_stale!
+        retry unless attempts > 1
+        raise
+      end
+    end
   end
 end

data/lib/kafka/consumer_group.rb

@@ -9,7 +9,7 @@ module Kafka
 
     def initialize(cluster:, logger:, group_id:, session_timeout:, retention_time:, instrumenter:)
       @cluster = cluster
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @group_id = group_id
       @session_timeout = session_timeout
       @instrumenter = instrumenter
@@ -122,6 +122,15 @@ module Kafka
       retry
     end
 
+    def to_s
+      "[#{@group_id}] {" + assigned_partitions.map { |topic, partitions|
+        partition_str = partitions.size > 5 ?
+                          "#{partitions[0..4].join(', ')}..." :
+                          partitions.join(', ')
+        "#{topic}: #{partition_str}"
+      }.join('; ') + '}:'
+    end
+
     private
 
     def join_group

data/lib/kafka/fetch_operation.rb

@@ -23,7 +23,7 @@ module Kafka
   class FetchOperation
     def initialize(cluster:, logger:, min_bytes: 1, max_bytes: 10485760, max_wait_time: 5)
       @cluster = cluster
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @min_bytes = min_bytes
       @max_bytes = max_bytes
       @max_wait_time = max_wait_time

data/lib/kafka/fetched_batch_generator.rb

@@ -10,7 +10,7 @@ module Kafka
     def initialize(topic, fetched_partition, offset, logger:)
       @topic = topic
       @fetched_partition = fetched_partition
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @offset = offset
     end
 

data/lib/kafka/fetched_offset_resolver.rb

@@ -3,7 +3,7 @@
 module Kafka
   class FetchedOffsetResolver
     def initialize(logger:)
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
     end
 
     def resolve!(broker, topics)

data/lib/kafka/fetcher.rb

@@ -8,7 +8,7 @@ module Kafka
 
     def initialize(cluster:, logger:, instrumenter:, max_queue_size:, group:)
       @cluster = cluster
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
       @max_queue_size = max_queue_size
       @group = group
@@ -55,7 +55,7 @@ module Kafka
         while @running
           loop
         end
-        @logger.info "Fetcher thread exited."
+        @logger.info "#{@group} Fetcher thread exited."
       end
       @thread.abort_on_exception = true
     end
@@ -94,6 +94,7 @@ module Kafka
     attr_reader :current_reset_counter
 
     def loop
+      @logger.push_tags(@group.to_s)
       @instrumenter.instrument("loop.fetcher", {
         queue_size: @queue.size,
       })
@@ -112,6 +113,8 @@ module Kafka
         @logger.warn "Reached max fetcher queue size (#{@max_queue_size}), sleeping 1s"
         sleep 1
       end
+    ensure
+      @logger.pop_tags
     end
 
     def handle_configure(min_bytes, max_bytes, max_wait_time)

data/lib/kafka/offset_manager.rb

@@ -13,7 +13,7 @@ module Kafka
       @cluster = cluster
       @group = group
       @fetcher = fetcher
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @commit_interval = commit_interval
       @commit_threshold = commit_threshold
 

data/lib/kafka/produce_operation.rb

@@ -37,7 +37,7 @@ module Kafka
       @required_acks = required_acks
       @ack_timeout = ack_timeout
       @compressor = compressor
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
     end
 

data/lib/kafka/producer.rb

@@ -130,7 +130,7 @@ module Kafka
     def initialize(cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
       @cluster = cluster
       @transaction_manager = transaction_manager
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
       @instrumenter = instrumenter
       @required_acks = required_acks == :all ? -1 : required_acks
       @ack_timeout = ack_timeout
@@ -150,6 +150,10 @@ module Kafka
       @pending_message_queue = PendingMessageQueue.new
     end
 
+    def to_s
+      "Producer #{@target_topics.to_a.join(', ')}"
+    end
+
     # Produces a message to the specified topic. Note that messages are buffered in
     # the producer until {#deliver_messages} is called.
     #
@@ -205,7 +209,7 @@ module Kafka
       # If the producer is in transactional mode, all the message production
       # must be used when the producer is currently in transaction
       if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
-        raise 'You must trigger begin_transaction before producing messages'
+        raise "Cannot produce to #{topic}: You must trigger begin_transaction before producing messages"
       end
 
       @target_topics.add(topic)
@@ -391,11 +395,11 @@ module Kafka
         if buffer_size.zero?
           break
         elsif attempt <= @max_retries
-          @logger.warn "Failed to send all messages; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
+          @logger.warn "Failed to send all messages to #{pretty_partitions}; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
 
           sleep @retry_backoff
         else
-          @logger.error "Failed to send all messages; keeping remaining messages in buffer"
+          @logger.error "Failed to send all messages to #{pretty_partitions}; keeping remaining messages in buffer"
           break
         end
       end
@@ -407,12 +411,14 @@ module Kafka
       end
 
       unless @buffer.empty?
-        partitions = @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
-
-        raise DeliveryFailed.new("Failed to send messages to #{partitions}", buffer_messages)
+        raise DeliveryFailed.new("Failed to send messages to #{pretty_partitions}", buffer_messages)
       end
     end
 
+    def pretty_partitions
+      @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
+    end
+
     def assign_partitions!
       failed_messages = []
       topics_with_failures = Set.new

data/lib/kafka/protocol/record_batch.rb

@@ -11,6 +11,7 @@ module Kafka
       CODEC_ID_MASK = 0b00000111
       IN_TRANSACTION_MASK = 0b00010000
       IS_CONTROL_BATCH_MASK = 0b00100000
+      TIMESTAMP_TYPE_MASK = 0b001000
 
       attr_reader :records, :first_offset, :first_timestamp, :partition_leader_epoch, :in_transaction, :is_control_batch, :last_offset_delta, :max_timestamp, :producer_id, :producer_epoch, :first_sequence
 
@@ -163,6 +164,7 @@ module Kafka
         codec_id = attributes & CODEC_ID_MASK
         in_transaction = (attributes & IN_TRANSACTION_MASK) > 0
         is_control_batch = (attributes & IS_CONTROL_BATCH_MASK) > 0
+        log_append_time = (attributes & TIMESTAMP_TYPE_MASK) != 0
 
         last_offset_delta = record_batch_decoder.int32
         first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
@@ -186,7 +188,7 @@ module Kafka
         until records_array_decoder.eof?
           record = Record.decode(records_array_decoder)
           record.offset = first_offset + record.offset_delta
-          record.create_time = first_timestamp + record.timestamp_delta
+          record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta
           records_array << record
         end
 

data/lib/kafka/sasl/gssapi.rb

@@ -7,7 +7,7 @@ module Kafka
       GSSAPI_CONFIDENTIALITY = false
 
       def initialize(logger:, principal:, keytab:)
-        @logger = logger
+        @logger = TaggedLogger.new(logger)
         @principal = principal
         @keytab = keytab
       end

data/lib/kafka/sasl/plain.rb

@@ -6,7 +6,7 @@ module Kafka
       PLAIN_IDENT = "PLAIN"
 
       def initialize(logger:, authzid:, username:, password:)
-        @logger = logger
+        @logger = TaggedLogger.new(logger)
         @authzid = authzid
         @username = username
         @password = password

data/lib/kafka/sasl/scram.rb

@@ -14,7 +14,7 @@ module Kafka
       def initialize(username:, password:, mechanism: 'sha256', logger:)
         @username = username
         @password = password
-        @logger = logger
+        @logger = TaggedLogger.new(logger)
 
         if mechanism
           @mechanism = MECHANISMS.fetch(mechanism) do

data/lib/kafka/sasl_authenticator.rb

@@ -9,7 +9,7 @@ module Kafka
     def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:,
                    sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:,
                    sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism:)
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
 
       @plain = Sasl::Plain.new(
         authzid: sasl_plain_authzid,

data/lib/kafka/tagged_logger.rb

@@ -0,0 +1,72 @@
+require 'forwardable'
+
+# Basic implementation of a tagged logger that matches the API of
+# ActiveSupport::TaggedLogging.
+
+module Kafka
+  module TaggedFormatter
+
+    def call(severity, timestamp, progname, msg)
+      super(severity, timestamp, progname, "#{tags_text}#{msg}")
+    end
+
+    def tagged(*tags)
+      new_tags = push_tags(*tags)
+      yield self
+    ensure
+      pop_tags(new_tags.size)
+    end
+
+    def push_tags(*tags)
+      tags.flatten.reject { |t| t.nil? || t.empty? }.tap do |new_tags|
+        current_tags.concat new_tags
+      end
+    end
+
+    def pop_tags(size = 1)
+      current_tags.pop size
+    end
+
+    def clear_tags!
+      current_tags.clear
+    end
+
+    def current_tags
+      # We use our object ID here to avoid conflicting with other instances
+      thread_key = @thread_key ||= "kafka_tagged_logging_tags:#{object_id}".freeze
+      Thread.current[thread_key] ||= []
+    end
+
+    def tags_text
+      tags = current_tags
+      if tags.any?
+        tags.collect { |tag| "[#{tag}] " }.join
+      end
+    end
+
+  end
+
+  module TaggedLogger
+    extend Forwardable
+    delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
+
+    def self.new(logger)
+      logger ||= Logger.new(nil)
+      return logger if logger.respond_to?(:push_tags) # already included
+      # Ensure we set a default formatter so we aren't extending nil!
+      logger.formatter ||= Logger::Formatter.new
+      logger.formatter.extend TaggedFormatter
+      logger.extend(self)
+    end
+
+    def tagged(*tags)
+      formatter.tagged(*tags) { yield self }
+    end
+
+    def flush
+      clear_tags!
+      super if defined?(super)
+    end
+  end
+
+end

data/lib/kafka/transaction_manager.rb

@@ -19,7 +19,7 @@ module Kafka
       transactional_timeout: DEFAULT_TRANSACTION_TIMEOUT
     )
       @cluster = cluster
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
 
       @transactional = transactional
       @transactional_id = transactional_id

data/lib/kafka/transaction_state_machine.rb

@@ -27,7 +27,7 @@ module Kafka
     def initialize(logger:)
       @state = UNINITIALIZED
       @mutex = Mutex.new
-      @logger = logger
+      @logger = TaggedLogger.new(logger)
     end
 
     def transition_to!(next_state)

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.5"
+  VERSION = "0.7.6.beta1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-15 00:00:00.000000000 Z
+date: 2019-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -426,6 +426,7 @@ files:
 - lib/kafka/ssl_context.rb
 - lib/kafka/ssl_socket_with_timeout.rb
 - lib/kafka/statsd.rb
+- lib/kafka/tagged_logger.rb
 - lib/kafka/transaction_manager.rb
 - lib/kafka/transaction_state_machine.rb
 - lib/kafka/version.rb
@@ -446,9 +447,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6