karafka 2.0.37 → 2.0.39

data/lib/karafka/processing/executors_buffer.rb

@@ -20,14 +20,13 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       # @param parallel_key [String] parallel group key
+      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Executor] consumer executor
-      def find_or_create(topic, partition, parallel_key)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition][parallel_key] ||= Executor.new(
+      def find_or_create(topic, partition, parallel_key, coordinator)
+        @buffer[topic][partition][parallel_key] ||= Executor.new(
           @subscription_group.id,
           @client,
-          ktopic
+          coordinator
         )
       end
 
@@ -37,9 +36,7 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def revoke(topic, partition)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition].clear
+        @buffer[topic][partition].clear
       end
 
       # Finds all the executors available for a given topic partition
@@ -48,9 +45,7 @@ module Karafka
       # @param partition [Integer] partition number
       # @return [Array<Executor>] executors in use for this topic + partition
       def find_all(topic, partition)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition].values
+        @buffer[topic][partition].values
       end
 
       # Iterates over all available executors and yields them together with topic and partition
@@ -59,11 +54,10 @@ module Karafka
       # @yieldparam [Integer] partition number
       # @yieldparam [Executor] given executor
       def each
-        @buffer.each do |ktopic, partitions|
-          partitions.each do |partition, executors|
-            executors.each do |_parallel_key, executor|
-              # We skip the parallel key here as it does not serve any value when iterating
-              yield(ktopic, partition, executor)
+        @buffer.each do |_, partitions|
+          partitions.each do |_, executors|
+            executors.each do |_, executor|
+              yield(executor)
             end
           end
         end
@@ -73,16 +67,6 @@ module Karafka
       def clear
         @buffer.clear
       end
-
-      private
-
-      # Finds topic based on its name
-      #
-      # @param topic [String] topic we're looking for
-      # @return [Karafka::Routing::Topic] topic we're interested in
-      def find_topic(topic)
-        @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
-      end
     end
   end
 end

data/lib/karafka/processing/jobs/consume.rb

@@ -12,19 +12,17 @@ module Karafka
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch
-        # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
         # @return [Consume]
-        def initialize(executor, messages, coordinator)
+        def initialize(executor, messages)
           @executor = executor
           @messages = messages
-          @coordinator = coordinator
           super()
         end
 
         # Runs all the preparation code on the executor that needs to happen before the job is
         # enqueued.
         def before_enqueue
-          executor.before_enqueue(@messages, @coordinator)
+          executor.before_enqueue(@messages)
         end
 
         # Runs the before consumption preparations on the executor

data/lib/karafka/processing/jobs/idle.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Type of job that we may use to run some extra handling that happens without the user
+      # related lifecycle event like consumption, revocation, etc.
+      class Idle < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Run the idle work via the executor
+        def call
+          executor.idle
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_builder.rb

@@ -7,10 +7,9 @@ module Karafka
     class JobsBuilder
       # @param executor [Karafka::Processing::Executor]
       # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Karafka::Processing::Jobs::Consume] consumption job
-      def consume(executor, messages, coordinator)
-        Jobs::Consume.new(executor, messages, coordinator)
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
       end
 
       # @param executor [Karafka::Processing::Executor]

data/lib/karafka/processing/result.rb

@@ -32,6 +32,11 @@ module Karafka
         @success = false
         @cause = cause
       end
+
+      # @return [Boolean] true if processing failed
+      def failure?
+        !@success
+      end
     end
   end
 end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -29,7 +29,7 @@ module Karafka
             retry_after_pause
           else
             coordinator.pause_tracker.reset
-            skippable_message = find_skippable_message
+            skippable_message, = find_skippable_message
             dispatch_to_dlq(skippable_message)
             # We can commit the offset here because we know that we skip it "forever" and
             # since AJ consumer commits the offset after each job, we also know that the

data/lib/karafka/processing/strategies/base.rb

@@ -32,6 +32,11 @@ module Karafka
           raise NotImplementedError, 'Implement in a subclass'
         end
 
+        # Idle run handling
+        def handle_idle
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
         # Revocation handling
         def handle_revoked
           raise NotImplementedError, 'Implement in a subclass'

data/lib/karafka/processing/strategies/default.rb

@@ -13,6 +13,51 @@ module Karafka
         # Apply strategy for a non-feature based flow
         FEATURES = %i[].freeze
 
+        # Marks message as consumed in an async way.
+        #
+        # @param message [Messages::Message] last successfully processed message.
+        # @return [Boolean] true if we were able to mark the offset, false otherwise.
+        #   False indicates that we were not able and that we have lost the partition.
+        #
+        # @note We keep track of this offset in case we would mark as consumed and got error when
+        #   processing another message. In case like this we do not pause on the message we've
+        #   already processed but rather at the next one. This applies to both sync and async
+        #   versions of this method.
+        def mark_as_consumed(message)
+          # Ignore earlier offsets than the one we already committed
+          return true if coordinator.seek_offset > message.offset
+
+          unless client.mark_as_consumed(message)
+            coordinator.revoke
+
+            return false
+          end
+
+          coordinator.seek_offset = message.offset + 1
+
+          true
+        end
+
+        # Marks message as consumed in a sync way.
+        #
+        # @param message [Messages::Message] last successfully processed message.
+        # @return [Boolean] true if we were able to mark the offset, false otherwise.
+        #   False indicates that we were not able and that we have lost the partition.
+        def mark_as_consumed!(message)
+          # Ignore earlier offsets than the one we already committed
+          return true if coordinator.seek_offset > message.offset
+
+          unless client.mark_as_consumed!(message)
+            coordinator.revoke
+
+            return false
+          end
+
+          coordinator.seek_offset = message.offset + 1
+
+          true
+        end
+
         # No actions needed for the standard flow here
         def handle_before_enqueue
           nil
@@ -63,6 +108,11 @@ module Karafka
           end
         end
 
+        # Code that should run on idle runs without messages available
+        def handle_idle
+          nil
+        end
+
         # We need to always un-pause the processing in case we have lost a given partition.
         # Otherwise the underlying librdkafka would not know we may want to continue processing and
         # the pause could in theory last forever

data/lib/karafka/processing/strategies/dlq.rb

@@ -34,7 +34,7 @@ module Karafka
             # We reset the pause to indicate we will now consider it as "ok".
             coordinator.pause_tracker.reset
 
-            skippable_message = find_skippable_message
+            skippable_message, = find_skippable_message
 
             # Send skippable message to the dql topic
             dispatch_to_dlq(skippable_message)
@@ -49,11 +49,20 @@ module Karafka
           end
         end
 
-        # Finds the message we want to skip
+        # Finds the message may want to skip (all, starting from first)
         # @private
+        # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+        #   information if this message was from marked offset or figured out via mom flow
         def find_skippable_message
-          skippable_message = messages.find { |message| message.offset == coordinator.seek_offset }
-          skippable_message || raise(Errors::SkipMessageNotFoundError, topic.name)
+          skippable_message = messages.find do |msg|
+            coordinator.marked? && msg.offset == coordinator.seek_offset
+          end
+
+          # If we don't have the message matching the last comitted offset, it means that
+          # user operates with manual offsets and we're beyond the batch in which things
+          # broke for the first time. Then we skip the first (as no markings) and we
+          # move on one by one.
+          skippable_message ? [skippable_message, true] : [messages.first, false]
         end
 
         # Moves the broken message into a separate queue defined via the settings

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -29,11 +29,16 @@ module Karafka
             # We reset the pause to indicate we will now consider it as "ok".
             coordinator.pause_tracker.reset
 
-            skippable_message = find_skippable_message
+            skippable_message, marked = find_skippable_message
+
             dispatch_to_dlq(skippable_message)
 
-            # We pause to backoff once just in case.
-            pause(coordinator.seek_offset, nil, false)
+            # Backoff and move forward
+            if marked
+              pause(coordinator.seek_offset, nil, false)
+            else
+              pause(skippable_message.offset + 1, nil, false)
+            end
           end
         end
       end

data/lib/karafka/processing/strategy_selector.rb

@@ -4,27 +4,44 @@ module Karafka
   module Processing
     # Selector of appropriate processing strategy matching topic combinations
     class StrategySelector
+      attr_reader :strategies
+
+      # Features we support in the OSS offering.
+      SUPPORTED_FEATURES = %i[
+        active_job
+        manual_offset_management
+        dead_letter_queue
+      ].freeze
+
       def initialize
         # We load them once for performance reasons not to do too many lookups
-        @available_strategies = Strategies
-                                .constants
-                                .delete_if { |k| k == :Base }
-                                .map { |k| Strategies.const_get(k) }
+        @strategies = find_all
       end
 
       # @param topic [Karafka::Routing::Topic] topic with settings based on which we find strategy
       # @return [Module] module with proper strategy
       def find(topic)
-        feature_set = [
-          topic.active_job? ? :active_job : nil,
-          topic.manual_offset_management? ? :manual_offset_management : nil,
-          topic.dead_letter_queue? ? :dead_letter_queue : nil
-        ].compact
+        feature_set = SUPPORTED_FEATURES.map do |feature_name|
+          topic.public_send("#{feature_name}?") ? feature_name : nil
+        end
+
+        feature_set.compact!
 
-        @available_strategies.find do |strategy|
+        @strategies.find do |strategy|
           strategy::FEATURES.sort == feature_set.sort
         end || raise(Errors::StrategyNotFoundError, topic.name)
       end
+
+      private
+
+      # @return [Array<Module>] available strategies
+      def find_all
+        Strategies
+          .constants
+          .delete_if { |k| k == :Base }
+          .map { |k| Strategies.const_get(k) }
+          .uniq
+      end
     end
   end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.37'
+  VERSION = '2.0.39'
 end

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.37
+  version: 2.0.39
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-03-20 00:00:00.000000000 Z
+date: 2023-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -228,40 +228,82 @@ files:
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
+- lib/karafka/pro/processing/filters/base.rb
+- lib/karafka/pro/processing/filters/delayer.rb
+- lib/karafka/pro/processing/filters/expirer.rb
+- lib/karafka/pro/processing/filters/throttler.rb
+- lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
-- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
-- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb
-- lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
-- lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb
-- lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
-- lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
-- lib/karafka/pro/processing/strategies/aj_mom.rb
-- lib/karafka/pro/processing/strategies/aj_mom_vp.rb
+- lib/karafka/pro/processing/strategies.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_mom.rb
+- lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj/mom.rb
+- lib/karafka/pro/processing/strategies/aj/mom_vp.rb
 - lib/karafka/pro/processing/strategies/base.rb
 - lib/karafka/pro/processing/strategies/default.rb
-- lib/karafka/pro/processing/strategies/dlq.rb
-- lib/karafka/pro/processing/strategies/dlq_lrj.rb
-- lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
-- lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb
-- lib/karafka/pro/processing/strategies/dlq_mom.rb
-- lib/karafka/pro/processing/strategies/dlq_vp.rb
-- lib/karafka/pro/processing/strategies/lrj.rb
-- lib/karafka/pro/processing/strategies/lrj_mom.rb
-- lib/karafka/pro/processing/strategies/lrj_vp.rb
-- lib/karafka/pro/processing/strategies/mom.rb
-- lib/karafka/pro/processing/strategies/vp.rb
+- lib/karafka/pro/processing/strategies/dlq/default.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb
+- lib/karafka/pro/processing/strategies/dlq/lrj.rb
+- lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb
+- lib/karafka/pro/processing/strategies/dlq/mom.rb
+- lib/karafka/pro/processing/strategies/dlq/vp.rb
+- lib/karafka/pro/processing/strategies/ftr/default.rb
+- lib/karafka/pro/processing/strategies/ftr/vp.rb
+- lib/karafka/pro/processing/strategies/lrj/default.rb
+- lib/karafka/pro/processing/strategies/lrj/ftr.rb
+- lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb
+- lib/karafka/pro/processing/strategies/lrj/mom.rb
+- lib/karafka/pro/processing/strategies/lrj/vp.rb
+- lib/karafka/pro/processing/strategies/mom/default.rb
+- lib/karafka/pro/processing/strategies/mom/ftr.rb
+- lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
 - lib/karafka/pro/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/pro/routing/features/delaying.rb
+- lib/karafka/pro/routing/features/delaying/config.rb
+- lib/karafka/pro/routing/features/delaying/contract.rb
+- lib/karafka/pro/routing/features/delaying/topic.rb
+- lib/karafka/pro/routing/features/expiring.rb
+- lib/karafka/pro/routing/features/expiring/config.rb
+- lib/karafka/pro/routing/features/expiring/contract.rb
+- lib/karafka/pro/routing/features/expiring/topic.rb
+- lib/karafka/pro/routing/features/filtering.rb
+- lib/karafka/pro/routing/features/filtering/config.rb
+- lib/karafka/pro/routing/features/filtering/contract.rb
+- lib/karafka/pro/routing/features/filtering/topic.rb
 - lib/karafka/pro/routing/features/long_running_job.rb
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contract.rb
 - lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/throttling.rb
+- lib/karafka/pro/routing/features/throttling/config.rb
+- lib/karafka/pro/routing/features/throttling/contract.rb
+- lib/karafka/pro/routing/features/throttling/topic.rb
 - lib/karafka/pro/routing/features/virtual_partitions.rb
 - lib/karafka/pro/routing/features/virtual_partitions/config.rb
 - lib/karafka/pro/routing/features/virtual_partitions/contract.rb
@@ -273,6 +315,7 @@ files:
 - lib/karafka/processing/executors_buffer.rb
 - lib/karafka/processing/jobs/base.rb
 - lib/karafka/processing/jobs/consume.rb
+- lib/karafka/processing/jobs/idle.rb
 - lib/karafka/processing/jobs/revoked.rb
 - lib/karafka/processing/jobs/shutdown.rb
 - lib/karafka/processing/jobs_builder.rb
@@ -335,6 +378,7 @@ files:
 - lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
 - log/.gitkeep
+- renovate.json
 homepage: https://karafka.io
 licenses:
 - LGPL-3.0
@@ -362,7 +406,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.10
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # DLQ enabled
-        # Long-Running Job enabled
-        # Manual offset management enabled
-        #
-        # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
-        # offset management works in this case.
-        module AjDlqLrjMom
-          # We can use the same code as for VP because non VP behaves like:
-          # - with one virtual partition
-          # - with "never ending" collapse
-          include AjDlqLrjMomVp
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            long_running_job
-            manual_offset_management
-            dead_letter_queue
-          ].freeze
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # DLQ enabled
-        # Long-Running Job enabled
-        # Manual offset management enabled
-        # Virtual Partitions enabled
-        #
-        # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
-        # offset management works in this case.
-        module AjDlqLrjMomVp
-          include AjDlqMomVp
-          include AjLrjMom
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            long_running_job
-            manual_offset_management
-            dead_letter_queue
-            virtual_partitions
-          ].freeze
-
-          # This strategy is pretty much as non VP one because of the collapse
-          def handle_after_consume
-            coordinator.on_finished do |last_group_message|
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                return if revoked?
-                return if Karafka::App.stopping?
-
-                # Since we have VP here we do not commit intermediate offsets and need to commit
-                # them here. We do commit in collapsed mode but this is generalized.
-                mark_as_consumed(last_group_message)
-
-                seek(coordinator.seek_offset) unless revoked?
-
-                resume
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              else
-                coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                mark_as_consumed(skippable_message)
-                pause(coordinator.seek_offset, nil, false)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end