karafka 2.0.38 → 2.0.40

data/lib/karafka/pro/routing/features/filtering.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Filtering provides a generic API allowing you to pre-filter messages before they are
+        # dispatched to jobs and processed.
+        #
+        # It allows for throttling, delayed jobs and other filtering implementations.
+        class Filtering < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/contract.rb

@@ -16,7 +16,7 @@ module Karafka
     module Routing
       module Features
         class LongRunningJob < Base
-          # Rules around manual offset management settings
+          # Rules around long-running job settings
           class Contract < Contracts::Base
             configure do |config|
               config.error_messages = YAML.safe_load(

data/lib/karafka/pro/routing/features/throttling/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Throttling feature configuration
+          Config = Struct.new(
+            :active,
+            :limit,
+            :interval,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Rules around throttling settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:throttling) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:interval) { |val| val.is_a?(Integer) && val.positive? }
+              required(:limit) do |val|
+                (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Topic throttling API extensions
+          module Topic
+            # @param limit [Integer] max messages to process in an time interval
+            # @param interval [Integer] time interval for processing
+            def throttling(
+              limit: Float::INFINITY,
+              interval: 60_000
+            )
+              # Those settings are used for validation
+              @throttling ||= begin
+                config = Config.new(
+                  active: limit != Float::INFINITY,
+                  limit: limit,
+                  interval: interval
+                )
+
+                # If someone defined throttling setup, we need to create appropriate filter for it
+                # and inject it via filtering feature
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Throttler.new(limit, interval) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#throttling` accepts
+            def throttle(**args)
+              throttling(**args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def throttling?
+              throttling.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                throttling: throttling.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Ability to throttle ingestion of data per topic partition
+        # Useful when we have fixed limit of things we can process in a given time period without
+        # getting into trouble. It can be used for example to:
+        #   - make sure we do not insert things to DB too fast
+        #   - make sure we do not dispatch HTTP requests to external resources too fast
+        #
+        # This feature is virtual. It materializes itself via the `Filtering` feature.
+        class Throttling < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinator.rb

@@ -10,57 +10,65 @@ module Karafka
     #   listener thread, but we go with thread-safe by default for all not to worry about potential
     #   future mistakes.
     class Coordinator
-      # @return [Karafka::TimeTrackers::Pause]
-      attr_reader :pause_tracker
-
-      attr_reader :seek_offset
+      attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      # @param topic [Karafka::Routing::Topic]
+      # @param partition [Integer]
       # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
-      def initialize(pause_tracker)
+      def initialize(topic, partition, pause_tracker)
+        @topic = topic
+        @partition = partition
         @pause_tracker = pause_tracker
         @revoked = false
         @consumptions = {}
         @running_jobs = 0
         @manual_pause = false
         @mutex = Mutex.new
+        @marked = false
+        @failure = false
       end
 
       # Starts the coordinator for given consumption jobs
       # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
       #   going to coordinate work. Not used with regular coordinator.
       def start(messages)
-        @mutex.synchronize do
-          @running_jobs = 0
-          # We need to clear the consumption results hash here, otherwise we could end up storing
-          # consumption results of consumer instances we no longer control
-          @consumptions.clear
-
-          # When starting to run, no pause is expected and no manual pause as well
-          @manual_pause = false
-
-          # We set it on the first encounter and never again, because then the offset setting
-          # should be up to the consumers logic (our or the end user)
-          # Seek offset needs to be always initialized as for case where manual offset management
-          # is turned on, we need to have reference to the first offset even in case of running
-          # multiple batches without marking any messages as consumed. Rollback needs to happen to
-          # the last place we know of or the last message + 1 that was marked
-          @seek_offset ||= messages.first.offset
-        end
+        @failure = false
+        @running_jobs = 0
+        # We need to clear the consumption results hash here, otherwise we could end up storing
+        # consumption results of consumer instances we no longer control
+        @consumptions.clear
+
+        # When starting to run, no pause is expected and no manual pause as well
+        @manual_pause = false
+
+        # We set it on the first encounter and never again, because then the offset setting
+        # should be up to the consumers logic (our or the end user)
+        # Seek offset needs to be always initialized as for case where manual offset management
+        # is turned on, we need to have reference to the first offset even in case of running
+        # multiple batches without marking any messages as consumed. Rollback needs to happen to
+        # the last place we know of or the last message + 1 that was marked
+        #
+        # It is however worth keeping in mind, that this may need to be used with `#marked?` to
+        # make sure that the first offset is an offset that has been marked.
+        @seek_offset ||= messages.first.offset
       end
 
       # @param offset [Integer] message offset
       def seek_offset=(offset)
-        @mutex.synchronize { @seek_offset = offset }
+        synchronize do
+          @marked = true
+          @seek_offset = offset
+        end
       end
 
       # Increases number of jobs that we handle with this coordinator
       def increment
-        @mutex.synchronize { @running_jobs += 1 }
+        synchronize { @running_jobs += 1 }
       end
 
       # Decrements number of jobs we handle at the moment
       def decrement
-        @mutex.synchronize do
+        synchronize do
           @running_jobs -= 1
 
           return @running_jobs unless @running_jobs.negative?
@@ -72,8 +80,10 @@ module Karafka
       end
 
       # Is all the consumption done and finished successfully for this coordinator
+      # We do not say we're successful until all work is done, because running work may still
+      # crash.
       def success?
-        @mutex.synchronize do
+        synchronize do
           @running_jobs.zero? && @consumptions.values.all?(&:success?)
         end
       end
@@ -81,7 +91,7 @@ module Karafka
       # Mark given consumption on consumer as successful
       # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
       def success!(consumer)
-        @mutex.synchronize do
+        synchronize do
           consumption(consumer).success!
         end
       end
@@ -90,11 +100,17 @@ module Karafka
       # @param consumer [Karafka::BaseConsumer] consumer that failed
       # @param error [StandardError] error that occurred
       def failure!(consumer, error)
-        @mutex.synchronize do
+        synchronize do
+          @failure = true
           consumption(consumer).failure!(error)
         end
       end
 
+      # @return [Boolean] true if any of work we were running failed
+      def failure?
+        @failure
+      end
+
       # Marks given coordinator for processing group as revoked
       #
       # This is invoked in two places:
@@ -105,7 +121,7 @@ module Karafka
       # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
       # processed until revocation jobs are done.
       def revoke
-        @mutex.synchronize { @revoked = true }
+        synchronize { @revoked = true }
       end
 
       # @return [Boolean] is the partition we are processing revoked or not
@@ -113,10 +129,18 @@ module Karafka
         @revoked
       end
 
+      # @return [Boolean] was the new seek offset assigned at least once. This is needed because
+      #   by default we assign seek offset of a first message ever, however this is insufficient
+      #   for DLQ in a scenario where the first message would be broken. We would never move
+      #   out of it and would end up in an endless loop.
+      def marked?
+        @marked
+      end
+
       # Store in the coordinator info, that this pause was done manually by the end user and not
       # by the system itself
       def manual_pause
-        @mutex.synchronize { @manual_pause = true }
+        @manual_pause = true
       end
 
       # @return [Boolean] are we in a pause that was initiated by the user
@@ -124,6 +148,12 @@ module Karafka
         @pause_tracker.paused? && @manual_pause
       end
 
+      # Allows to run synchronized (locked) code that can operate in between virtual partitions
+      # @param block [Proc] code we want to run in the synchronized mode
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
       private
 
       # @param consumer [Object] karafka consumer (normal or pro)

data/lib/karafka/processing/coordinators_buffer.rb

@@ -9,16 +9,20 @@ module Karafka
     # @note This buffer operates only from the listener loop, thus we do not have to make it
     #   thread-safe.
     class CoordinatorsBuffer
-      def initialize
+      # @param topics [Karafka::Routing::Topics]
+      def initialize(topics)
         @pauses_manager = Connection::PausesManager.new
         @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
         @coordinators = Hash.new { |h, k| h[k] = {} }
+        @topics = topics
       end
 
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def find_or_create(topic, partition)
         @coordinators[topic][partition] ||= @coordinator_class.new(
+          @topics.find(topic),
+          partition,
           @pauses_manager.fetch(topic, partition)
         )
       end

data/lib/karafka/processing/executor.rb

@@ -12,6 +12,10 @@ module Karafka
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
     class Executor
+      extend Forwardable
+
+      def_delegators :@coordinator, :topic, :partition
+
       # @return [String] unique id that we use to ensure, that we use for state tracking
       attr_reader :id
 
@@ -21,20 +25,17 @@ module Karafka
       # @return [Karafka::Messages::Messages] messages batch
       attr_reader :messages
 
-      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
-      # based on the topic settings defined by the end user
-      #
-      # @return [Karafka::Routing::Topic] topic of this executor
-      attr_reader :topic
+      # @return [Karafka::Processing::Coordinator] coordinator for this executor
+      attr_reader :coordinator
 
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
-      # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      def initialize(group_id, client, topic)
+      # @param coordinator [Karafka::Processing::Coordinator]
+      def initialize(group_id, client, coordinator)
         @id = SecureRandom.hex(6)
         @group_id = group_id
         @client = client
-        @topic = topic
+        @coordinator = coordinator
       end
 
       # Allows us to prepare the consumer in the listener thread prior to the job being send to
@@ -42,25 +43,21 @@ module Karafka
       # queue as it could cause starvation.
       #
       # @param messages [Array<Karafka::Messages::Message>]
-      # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
-      def before_enqueue(messages, coordinator)
-        # the moment we've received the batch or actually the moment we've enqueued it,
-        # but good enough
-        @enqueued_at = Time.now
-
+      def before_enqueue(messages)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
         # middle state, where re-creation of a consumer instance would occur only sometimes
         @consumer = nil unless ::Karafka::App.config.consumer_persistence
 
-        consumer.coordinator = coordinator
-
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
           messages,
-          @topic,
-          @enqueued_at
+          topic,
+          partition,
+          # the moment we've received the batch or actually the moment we've enqueued it,
+          # but good enough
+          Time.now
         )
 
         consumer.on_before_enqueue
@@ -82,6 +79,23 @@ module Karafka
         consumer.on_after_consume
       end
 
+      # Runs consumer idle operations
+      # This may include house-keeping or other state management changes that can occur but that
+      # not mean there are any new messages available for the end user to process
+      def idle
+        # Initializes the messages set in case idle operation would happen before any processing
+        # This prevents us from having no messages object at all as the messages object and
+        # its metadata may be used for statistics
+        consumer.messages ||= Messages::Builders::Messages.call(
+          [],
+          topic,
+          partition,
+          Time.now
+        )
+
+        consumer.on_idle
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -114,15 +128,17 @@ module Karafka
       # @return [Object] cached consumer instance
       def consumer
         @consumer ||= begin
-          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(@topic)
+          topic = @coordinator.topic
+
+          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
 
-          consumer = @topic.consumer_class.new
+          consumer = topic.consumer_class.new
           # We use singleton class as the same consumer class may be used to process different
           # topics with different settings
           consumer.singleton_class.include(strategy)
-          consumer.topic = @topic
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
+          consumer.coordinator = @coordinator
 
           consumer
         end

data/lib/karafka/processing/executors_buffer.rb

@@ -20,14 +20,13 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       # @param parallel_key [String] parallel group key
+      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Executor] consumer executor
-      def find_or_create(topic, partition, parallel_key)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition][parallel_key] ||= Executor.new(
+      def find_or_create(topic, partition, parallel_key, coordinator)
+        @buffer[topic][partition][parallel_key] ||= Executor.new(
           @subscription_group.id,
           @client,
-          ktopic
+          coordinator
         )
       end
 
@@ -37,9 +36,7 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def revoke(topic, partition)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition].clear
+        @buffer[topic][partition].clear
       end
 
       # Finds all the executors available for a given topic partition
@@ -48,9 +45,7 @@ module Karafka
       # @param partition [Integer] partition number
       # @return [Array<Executor>] executors in use for this topic + partition
       def find_all(topic, partition)
-        ktopic = find_topic(topic)
-
-        @buffer[ktopic][partition].values
+        @buffer[topic][partition].values
       end
 
       # Iterates over all available executors and yields them together with topic and partition
@@ -59,11 +54,10 @@ module Karafka
       # @yieldparam [Integer] partition number
       # @yieldparam [Executor] given executor
       def each
-        @buffer.each do |ktopic, partitions|
-          partitions.each do |partition, executors|
-            executors.each do |_parallel_key, executor|
-              # We skip the parallel key here as it does not serve any value when iterating
-              yield(ktopic, partition, executor)
+        @buffer.each do |_, partitions|
+          partitions.each do |_, executors|
+            executors.each do |_, executor|
+              yield(executor)
             end
           end
         end
@@ -73,16 +67,6 @@ module Karafka
       def clear
         @buffer.clear
       end
-
-      private
-
-      # Finds topic based on its name
-      #
-      # @param topic [String] topic we're looking for
-      # @return [Karafka::Routing::Topic] topic we're interested in
-      def find_topic(topic)
-        @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
-      end
     end
   end
 end

data/lib/karafka/processing/jobs/consume.rb

@@ -12,19 +12,17 @@ module Karafka
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch
-        # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
         # @return [Consume]
-        def initialize(executor, messages, coordinator)
+        def initialize(executor, messages)
           @executor = executor
           @messages = messages
-          @coordinator = coordinator
           super()
         end
 
         # Runs all the preparation code on the executor that needs to happen before the job is
         # enqueued.
         def before_enqueue
-          executor.before_enqueue(@messages, @coordinator)
+          executor.before_enqueue(@messages)
         end
 
         # Runs the before consumption preparations on the executor

data/lib/karafka/processing/jobs/idle.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Type of job that we may use to run some extra handling that happens without the user
+      # related lifecycle event like consumption, revocation, etc.
+      class Idle < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Run the idle work via the executor
+        def call
+          executor.idle
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_builder.rb

@@ -7,10 +7,9 @@ module Karafka
     class JobsBuilder
       # @param executor [Karafka::Processing::Executor]
       # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Karafka::Processing::Jobs::Consume] consumption job
-      def consume(executor, messages, coordinator)
-        Jobs::Consume.new(executor, messages, coordinator)
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
       end
 
       # @param executor [Karafka::Processing::Executor]

data/lib/karafka/processing/result.rb

@@ -32,6 +32,11 @@ module Karafka
         @success = false
         @cause = cause
       end
+
+      # @return [Boolean] true if processing failed
+      def failure?
+        !@success
+      end
     end
   end
 end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -29,7 +29,7 @@ module Karafka
             retry_after_pause
           else
             coordinator.pause_tracker.reset
-            skippable_message = find_skippable_message
+            skippable_message, = find_skippable_message
             dispatch_to_dlq(skippable_message)
             # We can commit the offset here because we know that we skip it "forever" and
             # since AJ consumer commits the offset after each job, we also know that the