karafka 2.2.2 → 2.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 440133344045600ff04e132d48e16b83ce492d06616383f02f3c282ec28cd239
-  data.tar.gz: e12def9a87bf4e731bc049f15352c86ed95bdcd970ac4081cc720bd268e99d26
+  metadata.gz: 7e6cb6a9b75c85409da5835992ea35b8db57e7f38906d31e2dd4e6345027a357
+  data.tar.gz: 40ceebe714cf3db25dcc33a8b5ff8ea1426101aaf18862be907298334beddc16
 SHA512:
-  metadata.gz: 2c9deb6fdd1172e4a40828d8650d016e0cd660e3c3f4a98ab9bf6abcb2ce77179206e7d898ddfce4379fea994c4f7e3840a2a5b768648f13aa11504efd55feb0
-  data.tar.gz: 3543d19c5066d144d0d9a15d366d522e5b0134a3df601ef01e416659d71d07d977fef2e71d170dfa5382f8ce496ed554dd004b45bf936d472b2aab75ef473ac5
+  metadata.gz: 5e0f780dc0d46e5917df86afba04cb9620547abfc188b550b7667035ed624ab5a36a2dd35e9437fe43d3be66868f409f9d9c3b71d61825b6830261de2a973e27
+  data.tar.gz: bc2182903ab8bc48ed431054f2bf23226277608b017b070d8a7cf697b700dc612bea38de3ff1659b481a788e507e1dba0a3972f15e9434bbba8ad0cb48ecb5f5

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Karafka framework changelog
 
+## 2.2.4 (2023-09-13)
+- [Enhancement] Compensate for potential Kafka cluster drifts vs consumer drift in batch metadata (#1611).
+
+## 2.2.3 (2023-09-12)
+- [Fix] Karafka admin time based offset lookup can break for one non-default partition.
+
 ## 2.2.2 (2023-09-11)
 - [Feature] Provide ability to define routing defaults.
 - [Maintenance] Require `karafka-core` `>=` `2.2.2`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.2)
+    karafka (2.2.4)
       karafka-core (>= 2.2.2, < 2.3.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)

data/docker-compose.yml

@@ -1,14 +1,14 @@
 version: '2'
 services:
   zookeeper:
-    container_name: karafka_21_zookeeper
+    container_name: karafka_22_zookeeper
     image: wurstmeister/zookeeper
     restart: on-failure
     ports:
       - '2181:2181'
 
   kafka:
-    container_name: karafka_21_kafka
+    container_name: karafka_22_kafka
     image: wurstmeister/kafka
     ports:
       - '9092:9092'

data/lib/karafka/admin.rb

@@ -257,7 +257,10 @@ module Karafka
           )
 
           real_offsets = consumer.offsets_for_times(tpl)
-          detected_offset = real_offsets.to_h.dig(name, partition)
+          detected_offset = real_offsets
+                            .to_h
+                            .fetch(name)
+                            .find { |p_data| p_data.partition == partition }
 
           detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
         else

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -26,13 +26,33 @@ module Karafka
               topic: topic.name,
               # We go with the assumption that the creation of the whole batch is the last message
               # creation time
-              created_at: messages.last&.timestamp || nil,
+              created_at: local_created_at(messages.last),
               # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
               # This needs to be set to a correct value prior to processing starting
               processed_at: nil
             )
           end
+
+          private
+
+          # Code that aligns the batch creation at into our local time. If time of current machine
+          # and the Kafka cluster drift, this helps not to allow this to leak into the framework.
+          #
+          # @param last_message [::Karafka::Messages::Message, nil] last message from the batch or
+          #   nil if no message
+          # @return [Time] batch creation time. Now if no messages (workless flow) or the last
+          #   message time as long as the message is not from the future
+          # @note Message can be from the future in case consumer machine and Kafka cluster drift
+          #   apart and the machine is behind the cluster.
+          def local_created_at(last_message)
+            now = ::Time.now
+
+            return now unless last_message
+
+            timestamp = last_message.timestamp
+            timestamp > now ? now : timestamp
+          end
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.2'
+  VERSION = '2.2.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-11 00:00:00.000000000 Z
+date: 2023-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core