karafka 2.1.4 → 2.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e54dc80fd5a3a93857fd6a3a175030f7cb2eee1f77668d6936b705b0789c4228
-  data.tar.gz: e7f3b02811432987048bc599c2c4a6f0444fe13d98ca4aa8e87cebd5aaa8037a
+  metadata.gz: a6994a6d579728a877f84c87086d093aae8a1f830b891fcb4904883085432fe4
+  data.tar.gz: 13b21009a471194a72971ca81ddc718e044bb96587db0e8f186974f554e9ec62
 SHA512:
-  metadata.gz: 871ed7a55421041b1b232c47b3da0f625b8f2114c00057daf883223d3da4dc16637c9f0b4714cebe4a5f33249beac9bd7abbe15ec122f5b1a45a3c953dcb1465
-  data.tar.gz: ac2cc50c3277bd0bac7e3a2af40cc6d5a9c3aa92fc8cc3d2b12b8645329d269d0ed0d80c462d2fa6a03a677213e84802a1dfe8fdc1c494348b1a8b4fd9ef8459
+  metadata.gz: e4711880bde1d2cd1cb34959f740459979b74ff4d28a671a232f88adbe7473cf67e366fc2b492fac761c572f3a6dfc147a59d46fc08e1c5e18df8ac5f108afdd
+  data.tar.gz: c094600c2bd421ce309c0125d60ea82ed0106d5ce4566b3bb8c1aab13c553e7bd2f6651b98029e42ac831b132563b2c502dd1c76defbf8307cd9bd2393b258f7

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Karafka framework changelog
 
+## 2.1.6 (2023-06-29)
+- [Improvement] Provide time support for iterator
+- [Improvement] Provide time support for admin `#read_topic`
+- [Improvement] Provide time support for consumer `#seek`.
+- [Improvement] Remove no longer needed locks for client operations.
+- [Improvement] Raise `Karafka::Errors::TopicNotFoundError` when trying to iterate over non-existing topic.
+- [Improvement] Ensure that Kafka multi-command operations run under mutex together.
+- [Change] Require `waterdrop` `>= 2.6.2`
+- [Change] Require `karafka-core` `>= 2.1.1`
+- [Refactor] Clean-up iterator code.
+- [Fix]  Improve performance in dev environment for a Rails app (juike)
+- [Fix] Rename `InvalidRealOffsetUsage` to `InvalidRealOffsetUsageError` to align with naming of other errors.
+- [Fix] Fix unstable spec.
+- [Fix] Fix a case where automatic `#seek` would overwrite manual seek of a user when running LRJ.
+- [Fix] Make sure, that user direct `#seek` and `#pause` operations take precedence over system actions.
+- [Fix] Make sure, that `#pause` and `#resume` with one underlying connection do not race-condition.
+
+## 2.1.5 (2023-06-19)
+- [Improvement] Drastically improve `#revoked?` response quality by checking the real time assignment lost state on librdkafka.
+- [Improvement] Improve eviction of saturated jobs that would run on already revoked assignments.
+- [Improvement] Expose `#commit_offsets` and `#commit_offsets!` methods in the consumer to provide ability to commit offsets directly to Kafka without having to mark new messages as consumed.
+- [Improvement] No longer skip offset commit when no messages marked as consumed as `librdkafka` has fixed the crashes there.
+- [Improvement] Remove no longer needed patches.
+- [Improvement] Ensure, that the coordinator revocation status is switched upon revocation detection when using `#revoked?`
+- [Improvement] Add benchmarks for marking as consumed (sync and async).
+- [Change] Require `karafka-core` `>= 2.1.0`
+- [Change] Require `waterdrop` `>= 2.6.1`
+
 ## 2.1.4 (2023-06-06)
 - [Fix] `processing_lag` and `consumption_lag` on empty batch fail on shutdown usage (#1475)
 

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.4)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    karafka (2.1.6)
+      karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.5.3, < 3.0.0)
+      waterdrop (>= 2.6.2, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.4.3)
-      activesupport (= 7.0.4.3)
+    activejob (7.0.5)
+      activesupport (= 7.0.5)
       globalid (>= 0.3.6)
-    activesupport (7.0.4.3)
+    activesupport (7.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,26 +28,26 @@ GEM
     ffi (1.15.5)
     globalid (1.1.0)
       activesupport (>= 5.0)
-    i18n (1.13.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.13)
+    karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12.3)
-    karafka-rdkafka (0.12.3)
+      karafka-rdkafka (>= 0.13.1, < 0.14.0)
+    karafka-rdkafka (0.13.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.5.2)
+    karafka-web (0.6.1)
       erubi (~> 1.4)
-      karafka (>= 2.0.40, < 3.0.0)
-      karafka-core (>= 2.0.12, < 3.0.0)
-      roda (~> 3.63)
+      karafka (>= 2.1.4, < 3.0.0)
+      karafka-core (>= 2.0.13, < 3.0.0)
+      roda (~> 3.68, >= 3.68)
       tilt (~> 2.0)
     mini_portile2 (2.8.2)
-    minitest (5.18.0)
-    rack (3.0.7)
+    minitest (5.18.1)
+    rack (3.0.8)
     rake (13.0.6)
-    roda (3.68.0)
+    roda (3.69.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -69,11 +69,11 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.2)
-    tilt (2.1.0)
+    tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.5.3)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    waterdrop (2.6.2)
+      karafka-core (>= 2.1.0, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)
 

data/karafka.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.13', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.1', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.5.3', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.2', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/admin.rb

@@ -18,6 +18,9 @@ module Karafka
     # retry after checking that the operation was finished or failed using external factor.
     MAX_WAIT_TIMEOUT = 1
 
+    # Max time for a TPL request. We increase it to compensate for remote clusters latency
+    TPL_REQUEST_TIMEOUT = 2_000
+
     # How many times should be try. 1 x 60 => 60 seconds wait in total
     MAX_ATTEMPTS = 60
 
@@ -34,7 +37,8 @@ module Karafka
       'enable.auto.commit': false
     }.freeze
 
-    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
+    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
+                     :MAX_ATTEMPTS
 
     class << self
       # Allows us to read messages from the topic
@@ -42,8 +46,9 @@ module Karafka
       # @param name [String, Symbol] topic name
       # @param partition [Integer] partition
       # @param count [Integer] how many messages we want to get at most
-      # @param start_offset [Integer] offset from which we should start. If -1 is provided
-      #   (default) we will start from the latest offset
+      # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset. If time is provided, the appropriate
+      #   offset will be resolved.
       # @param settings [Hash] kafka extra settings (optional)
       #
       # @return [Array<Karafka::Messages::Message>] array with messages
@@ -53,6 +58,9 @@ module Karafka
         low_offset, high_offset = nil
 
         with_consumer(settings) do |consumer|
+          # Convert the time offset (if needed)
+          start_offset = resolve_offset(consumer, name.to_s, partition, start_offset)
+
           low_offset, high_offset = consumer.query_watermark_offsets(name, partition)
 
           # Select offset dynamically if -1 or less
@@ -171,7 +179,9 @@ module Karafka
       # @return [Rdkafka::Metadata] cluster metadata info
       def cluster_info
         with_admin do |admin|
-          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+          admin.instance_variable_get('@native_kafka').with_inner do |inner|
+            Rdkafka::Metadata.new(inner)
+          end
         end
       end
 
@@ -241,6 +251,29 @@ module Karafka
 
         ::Rdkafka::Config.new(config_hash)
       end
+
+      # Resolves the offset if offset is in a time format. Otherwise returns the offset without
+      # resolving.
+      # @param consumer [::Rdkafka::Consumer]
+      # @param name [String, Symbol] expected topic name
+      # @param partition [Integer]
+      # @param offset [Integer, Time]
+      # @return [Integer] expected offset
+      def resolve_offset(consumer, name, partition, offset)
+        if offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            name, partition => offset
+          )
+
+          real_offsets = consumer.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          detected_offset = real_offsets.to_h.dig(name, partition)
+
+          detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        else
+          offset
+        end
+      end
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -70,6 +70,7 @@ module Karafka
     #
     # @return [Boolean] true if there was no exception, otherwise false.
     #
+    # @private
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
@@ -203,8 +204,15 @@ module Karafka
 
     # Seeks in the context of current topic and partition
     #
-    # @param offset [Integer] offset where we want to seek
-    def seek(offset)
+    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
+    #   want to seek.
+    # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
+    #   based seek. User seek operations should take precedence over system actions, hence we need
+    #   to know who invoked it.
+    # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
+    def seek(offset, manual_seek = true)
+      coordinator.manual_seek if manual_seek
+
       client.seek(
         Karafka::Messages::Seek.new(
           topic.name,
@@ -215,10 +223,18 @@ module Karafka
     end
 
     # @return [Boolean] true if partition was revoked from the current consumer
-    # @note We know that partition got revoked because when we try to mark message as consumed,
-    #   unless if is successful, it will return false
+    # @note There are two "levels" on which we can know that partition was revoked. First one is
+    #   when we loose the assignment involuntarily and second is when coordinator gets this info
+    #   after we poll with the rebalance callbacks. The first check allows us to get this notion
+    #   even before we poll but it gets reset when polling happens, hence we also need to switch
+    #   the coordinator state after the revocation (but prior to running more jobs)
     def revoked?
-      coordinator.revoked?
+      return true if coordinator.revoked?
+      return false unless client.assignment_lost?
+
+      coordinator.revoke
+
+      true
     end
 
     # @return [Boolean] are we retrying processing after an error. This can be used to provide a

data/lib/karafka/connection/client.rb

@@ -20,11 +20,14 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
+      # Max time for a TPL request. We increase it to compensate for remote clusters latency
+      TPL_REQUEST_TIMEOUT = 2_000
+
       # We want to make sure we never close several clients in the same moment to prevent
       # potential race conditions and other issues
       SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT
 
       # Creates a new consumer instance.
       #
@@ -35,15 +38,16 @@ module Karafka
         @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
-        @mutex = Mutex.new
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
         @rebalance_manager = RebalanceManager.new
         @kafka = build_consumer
-        # Marks if we need to offset. If we did not store offsets, we should not commit the offset
-        # position as it will crash rdkafka
-        @offsetting = false
+        # There are few operations that can happen in parallel from the listener threads as well
+        # as from the workers. They are not fully thread-safe because they may be composed out of
+        # few calls to Kafka or out of few internal state changes. That is why we mutex them.
+        # It mostly revolves around pausing and resuming.
+        @mutex = Mutex.new
         # We need to keep track of what we have paused for resuming
         # In case we loose partition, we still need to resume it, otherwise it won't be fetched
         # again if we get reassigned to it later on. We need to keep them as after revocation we
@@ -104,13 +108,15 @@ module Karafka
       #
       # @param message [Karafka::Messages::Message]
       def store_offset(message)
-        @mutex.synchronize do
-          internal_store_offset(message)
-        end
+        internal_store_offset(message)
+      end
+
+      # @return [Boolean] true if our current assignment has been lost involuntarily.
+      def assignment_lost?
+        @kafka.assignment_lost?
       end
 
       # Commits the offset on a current consumer in a non-blocking or blocking way.
-      # Ignoring a case where there would not be an offset (for example when rebalance occurs).
       #
       # @param async [Boolean] should the commit happen async or sync (async by default)
       # @return [Boolean] did committing was successful. It may be not, when we no longer own
@@ -118,13 +124,13 @@ module Karafka
       #
       # @note This will commit all the offsets for the whole consumer. In order to achieve
       #   granular control over where the offset should be for particular topic partitions, the
-      #   store_offset should be used to only store new offset when we want to to be flushed
+      #   store_offset should be used to only store new offset when we want them to be flushed
+      #
+      # @note This method for async may return `true` despite involuntary partition revocation as
+      #   it does **not** resolve to `lost_assignment?`. It returns only the commit state operation
+      #   result.
       def commit_offsets(async: true)
-        @mutex.lock
-
         internal_commit_offsets(async: async)
-      ensure
-        @mutex.unlock
       end
 
       # Commits offset in a synchronous way.
@@ -137,13 +143,11 @@ module Karafka
       # Seek to a particular message. The next poll on the topic/partition will return the
       # message at the given offset.
       #
-      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to.
+      #   It can have the time based offset.
+      # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
       def seek(message)
-        @mutex.lock
-
-        @kafka.seek(message)
-      ensure
-        @mutex.unlock
+        @mutex.synchronize { internal_seek(message) }
       end
 
       # Pauses given partition and moves back to last successful offset processed.
@@ -154,37 +158,34 @@ module Karafka
       #   be reprocessed after getting back to processing)
       # @note This will pause indefinitely and requires manual `#resume`
       def pause(topic, partition, offset)
-        @mutex.lock
-
-        # Do not pause if the client got closed, would not change anything
-        return if @closed
-
-        pause_msg = Messages::Seek.new(topic, partition, offset)
+        @mutex.synchronize do
+          # Do not pause if the client got closed, would not change anything
+          return if @closed
 
-        internal_commit_offsets(async: true)
+          pause_msg = Messages::Seek.new(topic, partition, offset)
 
-        # Here we do not use our cached tpls because we should not try to pause something we do
-        # not own anymore.
-        tpl = topic_partition_list(topic, partition)
+          internal_commit_offsets(async: true)
 
-        return unless tpl
+          # Here we do not use our cached tpls because we should not try to pause something we do
+          # not own anymore.
+          tpl = topic_partition_list(topic, partition)
 
-        Karafka.monitor.instrument(
-          'client.pause',
-          caller: self,
-          subscription_group: @subscription_group,
-          topic: topic,
-          partition: partition,
-          offset: offset
-        )
+          return unless tpl
 
-        @paused_tpls[topic][partition] = tpl
+          Karafka.monitor.instrument(
+            'client.pause',
+            caller: self,
+            subscription_group: @subscription_group,
+            topic: topic,
+            partition: partition,
+            offset: offset
+          )
 
-        @kafka.pause(tpl)
+          @paused_tpls[topic][partition] = tpl
 
-        @kafka.seek(pause_msg)
-      ensure
-        @mutex.unlock
+          @kafka.pause(tpl)
+          internal_seek(pause_msg)
+        end
       end
 
       # Resumes processing of a give topic partition after it was paused.
@@ -192,33 +193,31 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition
       def resume(topic, partition)
-        @mutex.lock
-
-        return if @closed
+        @mutex.synchronize do
+          return if @closed
 
-        # We now commit offsets on rebalances, thus we can do it async just to make sure
-        internal_commit_offsets(async: true)
+          # We now commit offsets on rebalances, thus we can do it async just to make sure
+          internal_commit_offsets(async: true)
 
-        # If we were not able, let's try to reuse the one we have (if we have)
-        tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
+          # If we were not able, let's try to reuse the one we have (if we have)
+          tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
-        return unless tpl
+          return unless tpl
 
-        # If we did not have it, it means we never paused this partition, thus no resume should
-        # happen in the first place
-        return unless @paused_tpls[topic].delete(partition)
+          # If we did not have it, it means we never paused this partition, thus no resume should
+          # happen in the first place
+          return unless @paused_tpls[topic].delete(partition)
 
-        Karafka.monitor.instrument(
-          'client.resume',
-          caller: self,
-          subscription_group: @subscription_group,
-          topic: topic,
-          partition: partition
-        )
+          Karafka.monitor.instrument(
+            'client.resume',
+            caller: self,
+            subscription_group: @subscription_group,
+            topic: topic,
+            partition: partition
+          )
 
-        @kafka.resume(tpl)
-      ensure
-        @mutex.unlock
+          @kafka.resume(tpl)
+        end
       end
 
       # Gracefully stops topic consumption.
@@ -235,9 +234,10 @@ module Karafka
       # @param [Karafka::Messages::Message] message that we want to mark as processed
       # @return [Boolean] true if successful. False if we no longer own given partition
       # @note This method won't trigger automatic offsets commits, rather relying on the offset
-      #   check-pointing trigger that happens with each batch processed
+      #   check-pointing trigger that happens with each batch processed. It will however check the
+      #   `librdkafka` assignment ownership to increase accuracy for involuntary revocations.
       def mark_as_consumed(message)
-        store_offset(message)
+        store_offset(message) && !assignment_lost?
       end
 
       # Marks a given message as consumed and commits the offsets in a blocking way.
@@ -254,12 +254,9 @@ module Karafka
       def reset
         close
 
-        @mutex.synchronize do
-          @closed = false
-          @offsetting = false
-          @paused_tpls.clear
-          @kafka = build_consumer
-        end
+        @closed = false
+        @paused_tpls.clear
+        @kafka = build_consumer
       end
 
       # Runs a single poll ignoring all the potential errors
@@ -281,7 +278,6 @@ module Karafka
       # @param message [Karafka::Messages::Message]
       # @return [Boolean] true if we could store the offset (if we still own the partition)
       def internal_store_offset(message)
-        @offsetting = true
         @kafka.store_offset(message)
         true
       rescue Rdkafka::RdkafkaError => e
@@ -294,11 +290,11 @@ module Karafka
       # Non thread-safe message committing method
       # @param async [Boolean] should the commit happen async or sync (async by default)
       # @return [Boolean] true if offset commit worked, false if we've lost the assignment
+      # @note We do **not** consider `no_offset` as any problem and we allow to commit offsets
+      #   even when no stored, because with sync commit, it refreshes the ownership state of the
+      #   consumer in a sync way.
       def internal_commit_offsets(async: true)
-        return true unless @offsetting
-
         @kafka.commit(nil, async)
-        @offsetting = false
 
         true
       rescue Rdkafka::RdkafkaError => e
@@ -317,28 +313,55 @@ module Karafka
         raise e
       end
 
+      # Non-mutexed seek that should be used only internally. Outside we expose `#seek` that is
+      # wrapped with a mutex.
+      #
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to.
+      #   It can have the time based offset.
+      def internal_seek(message)
+        # If the seek message offset is in a time format, we need to find the closest "real"
+        # offset matching before we seek
+        if message.offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            message.topic,
+            message.partition => message.offset
+          )
+
+          # Now we can overwrite the seek message offset with our resolved offset and we can
+          # then seek to the appropriate message
+          # We set the timeout to 2_000 to make sure that remote clusters handle this well
+          real_offsets = @kafka.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
+
+          # There always needs to be an offset. In case we seek into the future, where there
+          # are no offsets yet, we get -1 which indicates the most recent offset
+          # We should always detect offset, whether it is 0, -1 or a corresponding
+          message.offset = detected_partition&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        end
+
+        @kafka.seek(message)
+      end
+
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
         # Allow only one client to be closed at the same time
         SHUTDOWN_MUTEX.synchronize do
-          # Make sure that no other operations are happening on this client when we close it
-          @mutex.synchronize do
-            # Once client is closed, we should not close it again
-            # This could only happen in case of a race-condition when forceful shutdown happens
-            # and triggers this from a different thread
-            return if @closed
-
-            @closed = true
-
-            # Remove callbacks runners that were registered
-            ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-            ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
-
-            @kafka.close
-            @buffer.clear
-            # @note We do not clear rebalance manager here as we may still have revocation info
-            # here that we want to consider valid prior to running another reconnection
-          end
+          # Once client is closed, we should not close it again
+          # This could only happen in case of a race-condition when forceful shutdown happens
+          # and triggers this from a different thread
+          return if @closed
+
+          @closed = true
+
+          # Remove callbacks runners that were registered
+          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+          ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+          @kafka.close
+          @buffer.clear
+          # @note We do not clear rebalance manager here as we may still have revocation info
+          # here that we want to consider valid prior to running another reconnection
         end
       end
 

data/lib/karafka/connection/rebalance_manager.rb

@@ -49,9 +49,8 @@ module Karafka
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
       #
       # @private
-      # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_assigned(_, partitions)
+      def on_partitions_assigned(partitions)
         @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
@@ -59,9 +58,8 @@ module Karafka
       # Callback that kicks in inside of rdkafka, when partitions are revoked.
       #
       # @private
-      # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_revoked(_, partitions)
+      def on_partitions_revoked(partitions)
         @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end

data/lib/karafka/errors.rb

@@ -48,6 +48,9 @@ module Karafka
     StrategyNotFoundError = Class.new(BaseError)
 
     # This should never happen. Please open an issue if it does.
-    InvalidRealOffsetUsage = Class.new(BaseError)
+    InvalidRealOffsetUsageError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidTimeBasedOffsetError = Class.new(BaseError)
   end
 end

data/lib/karafka/messages/builders/message.rb

@@ -12,9 +12,6 @@ module Karafka
           # @param received_at [Time] moment when we've received the message
           # @return [Karafka::Messages::Message] message object with payload and metadata
           def call(kafka_message, topic, received_at)
-            # @see https://github.com/appsignal/rdkafka-ruby/issues/168
-            kafka_message.headers.transform_keys!(&:to_s)
-
             metadata = Karafka::Messages::Metadata.new(
               timestamp: kafka_message.timestamp,
               headers: kafka_message.headers,

data/lib/karafka/messages/seek.rb

@@ -4,6 +4,9 @@ module Karafka
   module Messages
     # "Fake" message that we use as an abstraction layer when seeking back.
     # This allows us to encapsulate a seek with a simple abstraction
+    #
+    # @note `#offset` can be either the offset value or the time of the offset
+    # (first equal or greater)
     Seek = Struct.new(:topic, :partition, :offset)
   end
 end