karafka 2.2.3 → 2.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f79fc941154e12d6f13cdcff51d8a7bb89728fac20d363fc8bd4241c72feea32
-  data.tar.gz: c5c3e9a61c44d7272fa34329c3d299682ae309d3f9765aa85e3d179c23e8329d
+  metadata.gz: 7e6cb6a9b75c85409da5835992ea35b8db57e7f38906d31e2dd4e6345027a357
+  data.tar.gz: 40ceebe714cf3db25dcc33a8b5ff8ea1426101aaf18862be907298334beddc16
 SHA512:
-  metadata.gz: b1c26344d3de73b003f43385f96f919e5e8a5834d62d03c65acbaa711d1528a5bd353059518bdab6f622c93068197376a4ab423140c9a291e54f38f85e4b76a7
-  data.tar.gz: d72e445c80beba05315605d12b2b9179b1eab8d271d31301afc48d17bd18cae52bd5bc0aff6f53483dae87e7c9853ca18adea72fc0700d7320fce2eb65d064eb
+  metadata.gz: 5e0f780dc0d46e5917df86afba04cb9620547abfc188b550b7667035ed624ab5a36a2dd35e9437fe43d3be66868f409f9d9c3b71d61825b6830261de2a973e27
+  data.tar.gz: bc2182903ab8bc48ed431054f2bf23226277608b017b070d8a7cf697b700dc612bea38de3ff1659b481a788e507e1dba0a3972f15e9434bbba8ad0cb48ecb5f5

data/CHANGELOG.md

@@ -1,6 +1,9 @@
 # Karafka framework changelog
 
-### 2.2.3 (2023-09-12)
+## 2.2.4 (2023-09-13)
+- [Enhancement] Compensate for potential Kafka cluster drifts vs consumer drift in batch metadata (#1611).
+
+## 2.2.3 (2023-09-12)
 - [Fix] Karafka admin time based offset lookup can break for one non-default partition.
 
 ## 2.2.2 (2023-09-11)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.3)
+    karafka (2.2.4)
       karafka-core (>= 2.2.2, < 2.3.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)

data/docker-compose.yml

@@ -1,14 +1,14 @@
 version: '2'
 services:
   zookeeper:
-    container_name: karafka_21_zookeeper
+    container_name: karafka_22_zookeeper
     image: wurstmeister/zookeeper
     restart: on-failure
     ports:
       - '2181:2181'
 
   kafka:
-    container_name: karafka_21_kafka
+    container_name: karafka_22_kafka
     image: wurstmeister/kafka
     ports:
       - '9092:9092'

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -26,13 +26,33 @@ module Karafka
               topic: topic.name,
               # We go with the assumption that the creation of the whole batch is the last message
               # creation time
-              created_at: messages.last&.timestamp || nil,
+              created_at: local_created_at(messages.last),
               # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
               # This needs to be set to a correct value prior to processing starting
               processed_at: nil
             )
           end
+
+          private
+
+          # Code that aligns the batch creation at into our local time. If time of current machine
+          # and the Kafka cluster drift, this helps not to allow this to leak into the framework.
+          #
+          # @param last_message [::Karafka::Messages::Message, nil] last message from the batch or
+          #   nil if no message
+          # @return [Time] batch creation time. Now if no messages (workless flow) or the last
+          #   message time as long as the message is not from the future
+          # @note Message can be from the future in case consumer machine and Kafka cluster drift
+          #   apart and the machine is behind the cluster.
+          def local_created_at(last_message)
+            now = ::Time.now
+
+            return now unless last_message
+
+            timestamp = last_message.timestamp
+            timestamp > now ? now : timestamp
+          end
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.3'
+  VERSION = '2.2.4'
 end

data.tar.gz.sig

@@ -1,4 +1,2 @@
-���N��8"zH�͵���%��InB`��,BJ8��x;����߅ϳ��m�<G~�p4��ز
-��J���yr��f�ٳպ<vy. �ŏ�)=� �Hb��ytCe�_�A���3	���
-z��|��c�U7yٕs�*���x�>lA���f�>��Qg+|ܔg���4����N�޺���K�D��m��&��-�����P/�J@�I|$_5;Yj�vH��Jh��"�v��jTƗ����(҃�\P�;��$\AU@E�mI�h�DzoQD�Q��
-

+ �K|%��ϸ{��s\T��"ɪ}�UC����r�s�+7�{��3B��%w�av��U�j�� u����֠����<}!6j~�Ki�V�0i�����tS�x<������V݁���Y�v���m���W��Eϸ^n��v�;-�Z9���_�U�E��bT�������ċM�B�M�C>Q"��>������̨wS±}�����7�+��'WJUG��+=�-Lv��ɻ=��?w���(Me	���W�M�AN�j�Yl�M{��Eo�=����pp�k�z!�k�<$��6Ϡ��e�y�5<��;&�y;� Q��=�Y��}
+���D���}�YŻ

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.3
+  version: 2.2.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-12 00:00:00.000000000 Z
+date: 2023-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core