karafka 2.0.39 → 2.0.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12fe8a47dc0ab16b0f7783424cd1aa043c2d2b228b4f4164f1cecefe604269d9
-  data.tar.gz: 9fa3bae282770dd67503c41ef4b73a27a38bfcff3bf472ddd63753d14d03614f
+  metadata.gz: 9a4dd6f2d6a1354ac085a5f7ff7de13ca8589d9e83f7fffe2f83c2e4501895a6
+  data.tar.gz: 55426141e6974b329f8e207e20d3acd3935ca025a58b3f1fc287be3c363dee8f
 SHA512:
-  metadata.gz: 9e6536c90a411a0b42337f73c00d9f454028366f42eabb1b7f40902181bcbcfd43258741d6fc51c6e29046b9ee1f8598755440d28a00ca96104a61a8095c20c2
-  data.tar.gz: be75dd1bfa744187f770f2e1f0deeedfba4f3fb1b824d6bab91f056f96e60a33498429e35ea22841404def0935e584db0df4289d2818631ace2e597d28785960
+  metadata.gz: d5bdbee5398cc6230d31d6915aef359071f68536b89bb7661ce3323a61b926966fe045fd15d3b19b648478a82ce631692302da63e2ccadc8c7c3b476649ae10e
+  data.tar.gz: 924af69aee02bb6a726e09d7f277c8a79dc18eccef6dccf51e62090dc956584ac242bb30b90d0edaec2c626d02960b445ed52e2142b34c20b8b59ce1339b0469

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Karafka framework changelog
 
+## 2.0.40 (2023-04-13)
+- [Improvement] Introduce `Karafka::Messages::Messages#empty?` method to handle Idle related cases where shutdown or revocation would be called on an empty messages set. This method allows for checking if there are any messages in the messages batch.
+- [Refactor] Require messages builder to accept partition and do not fetch it from messages.
+- [Refactor] Use empty messages set for internal APIs (Idle) (so there always is `Karafka::Messages::Messages`)
+- [Refactor] Allow for empty messages set initialization with -1001 and -1 on metadata (similar to `librdkafka`)
+
 ## 2.0.39 (2023-04-11)
 - **[Feature]** Provide ability to throttle/limit number of messages processed in a time unit (#1203)
 - **[Feature]** Provide Delayed Topics (#1000)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.39)
+    karafka (2.0.40)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -10,22 +10,23 @@ module Karafka
           #
           # @param messages [Array<Karafka::Messages::Message>] messages array
           # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @param partition [Integer] partition of this metadata
           # @param scheduled_at [Time] moment when the batch was scheduled for processing
           # @return [Karafka::Messages::BatchMetadata] batch metadata object
           #
           # @note We do not set `processed_at` as this needs to be assigned when the batch is
           #   picked up for processing.
-          def call(messages, topic, scheduled_at)
+          def call(messages, topic, partition, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
               size: messages.count,
-              first_offset: messages.first.offset,
-              last_offset: messages.last.offset,
+              first_offset: messages.first&.offset || -1001,
+              last_offset: messages.last&.offset || -1001,
               deserializer: topic.deserializer,
-              partition: messages.first.partition,
+              partition: partition,
               topic: topic.name,
               # We go with the assumption that the creation of the whole batch is the last message
               # creation time
-              created_at: messages.last.timestamp,
+              created_at: messages.last&.timestamp || nil,
               # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
               # This needs to be set to a correct value prior to processing starting

data/lib/karafka/messages/builders/messages.rb

@@ -11,14 +11,16 @@ module Karafka
           #
           # @param messages [Array<Karafka::Messages::Message>] karafka messages array
           # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @param partition [Integer] partition of those messages
           # @param received_at [Time] moment in time when the messages were received
           # @return [Karafka::Messages::Messages] messages batch object
-          def call(messages, topic, received_at)
+          def call(messages, topic, partition, received_at)
             # We cannot freeze the batch metadata because it is altered with the processed_at time
             # prior to the consumption. It is being frozen there
             metadata = BatchMetadata.call(
               messages,
               topic,
+              partition,
               received_at
             )
 

data/lib/karafka/messages/messages.rb

@@ -40,6 +40,11 @@ module Karafka
         map(&:raw_payload)
       end
 
+      # @return [Boolean] is the messages batch empty
+      def empty?
+        @messages_array.empty?
+      end
+
       # @return [Karafka::Messages::Message] first message
       def first
         @messages_array.first

data/lib/karafka/pro/processing/strategies/dlq/vp.rb

@@ -21,7 +21,8 @@ module Karafka
           #
           # In general because we collapse processing in virtual partitions to one on errors, there
           # is no special action that needs to be taken because we warranty that even with VPs
-          # on errors a retry collapses into a single state.
+          # on errors a retry collapses into a single state and from this single state we can
+          # mark as consumed the message that we are moving to the DLQ.
           module Vp
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/processing/executor.rb

@@ -44,10 +44,6 @@ module Karafka
       #
       # @param messages [Array<Karafka::Messages::Message>]
       def before_enqueue(messages)
-        # the moment we've received the batch or actually the moment we've enqueued it,
-        # but good enough
-        @enqueued_at = Time.now
-
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
@@ -57,8 +53,11 @@ module Karafka
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
           messages,
-          coordinator.topic,
-          @enqueued_at
+          topic,
+          partition,
+          # the moment we've received the batch or actually the moment we've enqueued it,
+          # but good enough
+          Time.now
         )
 
         consumer.on_before_enqueue
@@ -84,6 +83,16 @@ module Karafka
       # This may include house-keeping or other state management changes that can occur but that
       # not mean there are any new messages available for the end user to process
       def idle
+        # Initializes the messages set in case idle operation would happen before any processing
+        # This prevents us from having no messages object at all as the messages object and
+        # its metadata may be used for statistics
+        consumer.messages ||= Messages::Builders::Messages.call(
+          [],
+          topic,
+          partition,
+          Time.now
+        )
+
         consumer.on_idle
       end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.39'
+  VERSION = '2.0.40'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.39
+  version: 2.0.40
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-04-11 00:00:00.000000000 Z
+date: 2023-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core