karafka 2.0.41 → 2.1.4

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -28,6 +28,44 @@ module Karafka
               virtual_partitions
             ].freeze
 
+            # @param message [Karafka::Messages::Message] marks message as consumed
+            # @note This virtual offset management uses a regular default marking API underneath.
+            #   We do not alter the "real" marking API, as VPs are just one of many cases we want
+            #   to support and we do not want to impact them with collective offsets management
+            def mark_as_consumed(message)
+              return super if collapsed?
+
+              manager = coordinator.virtual_offset_manager
+
+              coordinator.synchronize do
+                manager.mark(message)
+                # If this is last marking on a finished flow, we can use the original
+                # last message and in order to do so, we need to mark all previous messages as
+                # consumed as otherwise the computed offset could be different
+                # We mark until our offset just in case of a DLQ flow or similar, where we do not
+                # want to mark all but until the expected location
+                manager.mark_until(message) if coordinator.finished?
+
+                return revoked? unless manager.markable?
+              end
+
+              manager.markable? ? super(manager.markable) : revoked?
+            end
+
+            # @param message [Karafka::Messages::Message] blocking marks message as consumed
+            def mark_as_consumed!(message)
+              return super if collapsed?
+
+              manager = coordinator.virtual_offset_manager
+
+              coordinator.synchronize do
+                manager.mark(message)
+                manager.mark_until(message) if coordinator.finished?
+              end
+
+              manager.markable? ? super(manager.markable) : revoked?
+            end
+
             # @return [Boolean] is the virtual processing collapsed in the context of given
             #   consumer.
             def collapsed?
@@ -45,6 +83,19 @@ module Karafka
             def failing?
               coordinator.failure?
             end
+
+            private
+
+            # Prior to adding work to the queue, registers all the messages offsets into the
+            # virtual offset group.
+            #
+            # @note This can be done without the mutex, because it happens from the same thread
+            #   for all the work (listener thread)
+            def handle_before_enqueue
+              coordinator.virtual_offset_manager.register(
+                messages.map(&:offset)
+              )
+            end
           end
         end
       end

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manager that keeps track of our offsets with the virtualization layer that are local
+      # to given partition assignment. It allows for easier offset management for virtual
+      # virtual partition cases as it provides us ability to mark as consumed and move the
+      # real offset behind as expected.
+      #
+      # @note We still use the regular coordinator "real" offset management as we want to have
+      #   them as separated as possible because the real seek offset management is also used for
+      #   pausing, filtering and others and should not be impacted by the virtual one
+      #
+      # @note This manager is **not** thread-safe by itself. It should operate from coordinator
+      #   locked locations.
+      class VirtualOffsetManager
+        attr_reader :groups
+
+        # @param topic [String]
+        # @param partition [Integer]
+        #
+        # @note We need topic and partition because we use a seek message (virtual) for real offset
+        #   management. We could keep real message reference but this can be memory consuming
+        #   and not worth it.
+        def initialize(topic, partition)
+          @topic = topic
+          @partition = partition
+          @groups = []
+          @marked = {}
+          @real_offset = -1
+        end
+
+        # Clears the manager for a next collective operation
+        def clear
+          @groups.clear
+          @marked = {}
+          @real_offset = -1
+        end
+
+        # Registers an offset group coming from one virtual consumer. In order to move the real
+        # underlying offset accordingly, we need to make sure to track the virtual consumers
+        # offsets groups independently and only materialize the end result.
+        #
+        # @param offsets_group [Array<Integer>] offsets from one virtual consumer
+        def register(offsets_group)
+          @groups << offsets_group
+
+          offsets_group.each { |offset| @marked[offset] = false }
+        end
+
+        # Marks given message as marked (virtually consumed).
+        # We mark given message offset and other earlier offsets from the same group as done
+        # and we can refresh our real offset representation based on that as it might have changed
+        # to a newer real offset.
+        # @param message [Karafka::Messages::Message] message coming from VP we want to mark
+        def mark(message)
+          offset = message.offset
+
+          group = @groups.find { |reg_group| reg_group.include?(offset) }
+
+          # This case can happen when someone uses MoM and wants to mark message from a previous
+          # batch as consumed. We can add it, since the real offset refresh will point to it
+          unless group
+            group = [offset]
+            @groups << group
+          end
+
+          position = group.index(offset)
+
+          # Mark all previous messages from the same group also as virtually consumed
+          group[0..position].each do |markable_offset|
+            @marked[markable_offset] = true
+          end
+
+          # Recompute the real offset representation
+          materialize_real_offset
+        end
+
+        # Mark all from all groups including the `message`.
+        # Useful when operating in a collapsed state for marking
+        # @param message [Karafka::Messages::Message]
+        def mark_until(message)
+          mark(message)
+
+          @groups.each do |group|
+            group.each do |offset|
+              next if offset > message.offset
+
+              @marked[offset] = true
+            end
+          end
+
+          materialize_real_offset
+        end
+
+        # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
+        def marked
+          @marked.select { |_, status| status }.map(&:first).sort
+        end
+
+        # Is there a real offset we can mark as consumed
+        # @return [Boolean]
+        def markable?
+          !@real_offset.negative?
+        end
+
+        # @return [Messages::Seek] markable message for real offset marking
+        def markable
+          raise Errors::InvalidRealOffsetUsage unless markable?
+
+          Messages::Seek.new(
+            @topic,
+            @partition,
+            @real_offset
+          )
+        end
+
+        private
+
+        # Recomputes the biggest possible real offset we can have.
+        # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+        # consumed because this will be the collective offset.
+        def materialize_real_offset
+          @marked.to_a.sort_by(&:first).each do |offset, marked|
+            break unless marked
+
+            @real_offset = offset
+          end
+
+          @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -44,23 +44,6 @@ module Karafka
 
               [[%i[virtual_partitions partitioner], :respond_to_call]]
             end
-
-            # Make sure that manual offset management is not used together with Virtual Partitions
-            # This would not make any sense as there would be edge cases related to skipping
-            # messages even if there were errors.
-            virtual do |data, errors|
-              next unless errors.empty?
-
-              virtual_partitions = data[:virtual_partitions]
-              manual_offset_management = data[:manual_offset_management]
-              active_job = data[:active_job]
-
-              next unless virtual_partitions[:active]
-              next unless manual_offset_management[:active]
-              next if active_job[:active]
-
-              [[%i[manual_offset_management], :not_with_virtual_partitions]]
-            end
           end
         end
       end

data/lib/karafka/processing/strategies/default.rb

@@ -26,6 +26,7 @@ module Karafka
         def mark_as_consumed(message)
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
+          return false if revoked?
 
           unless client.mark_as_consumed(message)
             coordinator.revoke
@@ -46,6 +47,7 @@ module Karafka
         def mark_as_consumed!(message)
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
+          return false if revoked?
 
           unless client.mark_as_consumed!(message)
             coordinator.revoke

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -29,16 +29,18 @@ module Karafka
             # We reset the pause to indicate we will now consider it as "ok".
             coordinator.pause_tracker.reset
 
-            skippable_message, marked = find_skippable_message
+            skippable_message, = find_skippable_message
 
             dispatch_to_dlq(skippable_message)
 
-            # Backoff and move forward
-            if marked
-              pause(coordinator.seek_offset, nil, false)
-            else
-              pause(skippable_message.offset + 1, nil, false)
-            end
+            # Save the next offset we want to go with after moving given message to DLQ
+            # Without this, we would not be able to move forward and we would end up
+            # in an infinite loop trying to un-pause from the message we've already processed
+            # Of course, since it's a MoM a rebalance or kill, will move it back as no
+            # offsets are being committed
+            coordinator.seek_offset = skippable_message.offset + 1
+
+            pause(coordinator.seek_offset, nil, false)
           end
         end
       end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.41'
+  VERSION = '2.1.4'
 end

data/lib/karafka.rb

@@ -114,6 +114,11 @@ end
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
+# Do not load CurrentAttributes components as they will be loaded if needed
+# @note We have to exclude both the .rb file as well as the whole directory so users can require
+# current attributes only when needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes'))
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes.rb'))
 # Do not load Railtie. It will load if after everything is ready, so we don't have to load any
 # Karafka components when we require this railtie. Railtie needs to be loaded last.
 loader.ignore(Karafka.gem_root.join('lib/karafka/railtie'))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.41
+  version: 2.1.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-04-19 00:00:00.000000000 Z
+date: 2023-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
+        version: 2.0.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
+        version: 2.0.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.10
+        version: 2.5.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.10
+        version: 2.5.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -156,6 +156,9 @@ files:
 - lib/active_job/queue_adapters/karafka_adapter.rb
 - lib/karafka.rb
 - lib/karafka/active_job/consumer.rb
+- lib/karafka/active_job/current_attributes.rb
+- lib/karafka/active_job/current_attributes/loading.rb
+- lib/karafka/active_job/current_attributes/persistence.rb
 - lib/karafka/active_job/dispatcher.rb
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
@@ -197,9 +200,9 @@ files:
 - lib/karafka/instrumentation/notifications.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
-- lib/karafka/instrumentation/vendors/datadog/listener.rb
 - lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb
+- lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -216,7 +219,6 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
-- lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/encryption.rb
 - lib/karafka/pro/encryption/cipher.rb
 - lib/karafka/pro/encryption/contracts/config.rb
@@ -233,6 +235,7 @@ files:
 - lib/karafka/pro/processing/filters/delayer.rb
 - lib/karafka/pro/processing/filters/expirer.rb
 - lib/karafka/pro/processing/filters/throttler.rb
+- lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
@@ -262,26 +265,35 @@ files:
 - lib/karafka/pro/processing/strategies/dlq/ftr.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/lrj.rb
 - lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/mom.rb
+- lib/karafka/pro/processing/strategies/dlq/mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/vp.rb
 - lib/karafka/pro/processing/strategies/ftr/default.rb
 - lib/karafka/pro/processing/strategies/ftr/vp.rb
 - lib/karafka/pro/processing/strategies/lrj/default.rb
 - lib/karafka/pro/processing/strategies/lrj/ftr.rb
 - lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb
 - lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb
 - lib/karafka/pro/processing/strategies/lrj/mom.rb
+- lib/karafka/pro/processing/strategies/lrj/mom_vp.rb
 - lib/karafka/pro/processing/strategies/lrj/vp.rb
 - lib/karafka/pro/processing/strategies/mom/default.rb
 - lib/karafka/pro/processing/strategies/mom/ftr.rb
+- lib/karafka/pro/processing/strategies/mom/ftr_vp.rb
+- lib/karafka/pro/processing/strategies/mom/vp.rb
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
+- lib/karafka/pro/processing/virtual_offset_manager.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
 - lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'metrics_listener'
-
-module Karafka
-  module Instrumentation
-    # Namespace for vendor specific instrumentation
-    module Vendors
-      # Datadog specific instrumentation
-      module Datadog
-        # Alias to keep backwards compatibility
-        Listener = MetricsListener
-      end
-    end
-  end
-end