karafka 2.0.37 → 2.0.39

data/lib/karafka/pro/routing/features/filtering/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature configuration
+          Config = Struct.new(:factories, keyword_init: true) do
+            # @return [Boolean] is this feature in use. Are any filters defined
+            def active?
+              !factories.empty?
+            end
+
+            # @return [Array<Object>] array of filters applicable to a topic partition
+            def filters
+              factories.map(&:call)
+            end
+
+            # @return [Hash] this config hash
+            def to_h
+              super.merge(active: active?)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Contract to validate configuration of the filtering feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:filtering) do
+              required(:active) { |val| [true, false].include?(val) }
+
+              required(:factories) do |val|
+                val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature topic extensions
+          module Topic
+            # @param factory [#call, nil] Callable that can produce new filters instances per
+            #   assigned topic partition. nil as default so this feature is disabled
+            def filter(factory = nil)
+              @filtering ||= Config.new(factories: [])
+              @filtering.factories << factory if factory
+              @filtering
+            end
+
+            # @param args [Array] Anything `#filter` accepts
+            # @return [Filtering::Config] alias to match the naming API for features
+            def filtering(*args)
+              filter(*args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def filtering?
+              filtering.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                filtering: filtering.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Filtering provides a generic API allowing you to pre-filter messages before they are
+        # dispatched to jobs and processed.
+        #
+        # It allows for throttling, delayed jobs and other filtering implementations.
+        class Filtering < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/contract.rb

@@ -16,7 +16,7 @@ module Karafka
     module Routing
       module Features
         class LongRunningJob < Base
-          # Rules around manual offset management settings
+          # Rules around long-running job settings
           class Contract < Contracts::Base
             configure do |config|
               config.error_messages = YAML.safe_load(

data/lib/karafka/pro/routing/features/throttling/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Throttling feature configuration
+          Config = Struct.new(
+            :active,
+            :limit,
+            :interval,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Rules around throttling settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:throttling) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:interval) { |val| val.is_a?(Integer) && val.positive? }
+              required(:limit) do |val|
+                (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Topic throttling API extensions
+          module Topic
+            # @param limit [Integer] max messages to process in an time interval
+            # @param interval [Integer] time interval for processing
+            def throttling(
+              limit: Float::INFINITY,
+              interval: 60_000
+            )
+              # Those settings are used for validation
+              @throttling ||= begin
+                config = Config.new(
+                  active: limit != Float::INFINITY,
+                  limit: limit,
+                  interval: interval
+                )
+
+                # If someone defined throttling setup, we need to create appropriate filter for it
+                # and inject it via filtering feature
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Throttler.new(limit, interval) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#throttling` accepts
+            def throttle(**args)
+              throttling(**args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def throttling?
+              throttling.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                throttling: throttling.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Ability to throttle ingestion of data per topic partition
+        # Useful when we have fixed limit of things we can process in a given time period without
+        # getting into trouble. It can be used for example to:
+        #   - make sure we do not insert things to DB too fast
+        #   - make sure we do not dispatch HTTP requests to external resources too fast
+        #
+        # This feature is virtual. It materializes itself via the `Filtering` feature.
+        class Throttling < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinator.rb

@@ -10,57 +10,65 @@ module Karafka
     #   listener thread, but we go with thread-safe by default for all not to worry about potential
     #   future mistakes.
     class Coordinator
-      # @return [Karafka::TimeTrackers::Pause]
-      attr_reader :pause_tracker
-
-      attr_reader :seek_offset
+      attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      # @param topic [Karafka::Routing::Topic]
+      # @param partition [Integer]
       # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
-      def initialize(pause_tracker)
+      def initialize(topic, partition, pause_tracker)
+        @topic = topic
+        @partition = partition
         @pause_tracker = pause_tracker
         @revoked = false
         @consumptions = {}
         @running_jobs = 0
         @manual_pause = false
         @mutex = Mutex.new
+        @marked = false
+        @failure = false
       end
 
       # Starts the coordinator for given consumption jobs
       # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
       #   going to coordinate work. Not used with regular coordinator.
       def start(messages)
-        @mutex.synchronize do
-          @running_jobs = 0
-          # We need to clear the consumption results hash here, otherwise we could end up storing
-          # consumption results of consumer instances we no longer control
-          @consumptions.clear
-
-          # When starting to run, no pause is expected and no manual pause as well
-          @manual_pause = false
-
-          # We set it on the first encounter and never again, because then the offset setting
-          # should be up to the consumers logic (our or the end user)
-          # Seek offset needs to be always initialized as for case where manual offset management
-          # is turned on, we need to have reference to the first offset even in case of running
-          # multiple batches without marking any messages as consumed. Rollback needs to happen to
-          # the last place we know of or the last message + 1 that was marked
-          @seek_offset ||= messages.first.offset
-        end
+        @failure = false
+        @running_jobs = 0
+        # We need to clear the consumption results hash here, otherwise we could end up storing
+        # consumption results of consumer instances we no longer control
+        @consumptions.clear
+
+        # When starting to run, no pause is expected and no manual pause as well
+        @manual_pause = false
+
+        # We set it on the first encounter and never again, because then the offset setting
+        # should be up to the consumers logic (our or the end user)
+        # Seek offset needs to be always initialized as for case where manual offset management
+        # is turned on, we need to have reference to the first offset even in case of running
+        # multiple batches without marking any messages as consumed. Rollback needs to happen to
+        # the last place we know of or the last message + 1 that was marked
+        #
+        # It is however worth keeping in mind, that this may need to be used with `#marked?` to
+        # make sure that the first offset is an offset that has been marked.
+        @seek_offset ||= messages.first.offset
       end
 
       # @param offset [Integer] message offset
       def seek_offset=(offset)
-        @mutex.synchronize { @seek_offset = offset }
+        synchronize do
+          @marked = true
+          @seek_offset = offset
+        end
       end
 
       # Increases number of jobs that we handle with this coordinator
       def increment
-        @mutex.synchronize { @running_jobs += 1 }
+        synchronize { @running_jobs += 1 }
       end
 
       # Decrements number of jobs we handle at the moment
       def decrement
-        @mutex.synchronize do
+        synchronize do
           @running_jobs -= 1
 
           return @running_jobs unless @running_jobs.negative?
@@ -72,8 +80,10 @@ module Karafka
       end
 
       # Is all the consumption done and finished successfully for this coordinator
+      # We do not say we're successful until all work is done, because running work may still
+      # crash.
       def success?
-        @mutex.synchronize do
+        synchronize do
           @running_jobs.zero? && @consumptions.values.all?(&:success?)
         end
       end
@@ -81,7 +91,7 @@ module Karafka
       # Mark given consumption on consumer as successful
       # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
       def success!(consumer)
-        @mutex.synchronize do
+        synchronize do
           consumption(consumer).success!
         end
       end
@@ -90,11 +100,17 @@ module Karafka
       # @param consumer [Karafka::BaseConsumer] consumer that failed
       # @param error [StandardError] error that occurred
       def failure!(consumer, error)
-        @mutex.synchronize do
+        synchronize do
+          @failure = true
           consumption(consumer).failure!(error)
         end
       end
 
+      # @return [Boolean] true if any of work we were running failed
+      def failure?
+        @failure
+      end
+
       # Marks given coordinator for processing group as revoked
       #
       # This is invoked in two places:
@@ -105,7 +121,7 @@ module Karafka
       # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
       # processed until revocation jobs are done.
       def revoke
-        @mutex.synchronize { @revoked = true }
+        synchronize { @revoked = true }
       end
 
       # @return [Boolean] is the partition we are processing revoked or not
@@ -113,10 +129,18 @@ module Karafka
         @revoked
       end
 
+      # @return [Boolean] was the new seek offset assigned at least once. This is needed because
+      #   by default we assign seek offset of a first message ever, however this is insufficient
+      #   for DLQ in a scenario where the first message would be broken. We would never move
+      #   out of it and would end up in an endless loop.
+      def marked?
+        @marked
+      end
+
       # Store in the coordinator info, that this pause was done manually by the end user and not
       # by the system itself
       def manual_pause
-        @mutex.synchronize { @manual_pause = true }
+        @manual_pause = true
       end
 
       # @return [Boolean] are we in a pause that was initiated by the user
@@ -124,6 +148,12 @@ module Karafka
         @pause_tracker.paused? && @manual_pause
       end
 
+      # Allows to run synchronized (locked) code that can operate in between virtual partitions
+      # @param block [Proc] code we want to run in the synchronized mode
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
       private
 
       # @param consumer [Object] karafka consumer (normal or pro)

data/lib/karafka/processing/coordinators_buffer.rb

@@ -9,16 +9,20 @@ module Karafka
     # @note This buffer operates only from the listener loop, thus we do not have to make it
     #   thread-safe.
     class CoordinatorsBuffer
-      def initialize
+      # @param topics [Karafka::Routing::Topics]
+      def initialize(topics)
         @pauses_manager = Connection::PausesManager.new
         @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
         @coordinators = Hash.new { |h, k| h[k] = {} }
+        @topics = topics
       end
 
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def find_or_create(topic, partition)
         @coordinators[topic][partition] ||= @coordinator_class.new(
+          @topics.find(topic),
+          partition,
           @pauses_manager.fetch(topic, partition)
         )
       end

data/lib/karafka/processing/executor.rb

@@ -12,6 +12,10 @@ module Karafka
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
     class Executor
+      extend Forwardable
+
+      def_delegators :@coordinator, :topic, :partition
+
       # @return [String] unique id that we use to ensure, that we use for state tracking
       attr_reader :id
 
@@ -21,20 +25,17 @@ module Karafka
       # @return [Karafka::Messages::Messages] messages batch
       attr_reader :messages
 
-      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
-      # based on the topic settings defined by the end user
-      #
-      # @return [Karafka::Routing::Topic] topic of this executor
-      attr_reader :topic
+      # @return [Karafka::Processing::Coordinator] coordinator for this executor
+      attr_reader :coordinator
 
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
-      # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      def initialize(group_id, client, topic)
+      # @param coordinator [Karafka::Processing::Coordinator]
+      def initialize(group_id, client, coordinator)
         @id = SecureRandom.hex(6)
         @group_id = group_id
         @client = client
-        @topic = topic
+        @coordinator = coordinator
       end
 
       # Allows us to prepare the consumer in the listener thread prior to the job being send to
@@ -42,8 +43,7 @@ module Karafka
       # queue as it could cause starvation.
       #
       # @param messages [Array<Karafka::Messages::Message>]
-      # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
-      def before_enqueue(messages, coordinator)
+      def before_enqueue(messages)
         # the moment we've received the batch or actually the moment we've enqueued it,
         # but good enough
         @enqueued_at = Time.now
@@ -54,12 +54,10 @@ module Karafka
         # middle state, where re-creation of a consumer instance would occur only sometimes
         @consumer = nil unless ::Karafka::App.config.consumer_persistence
 
-        consumer.coordinator = coordinator
-
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
           messages,
-          @topic,
+          coordinator.topic,
           @enqueued_at
         )
 
@@ -82,6 +80,13 @@ module Karafka
         consumer.on_after_consume
       end
 
+      # Runs consumer idle operations
+      # This may include house-keeping or other state management changes that can occur but that
+      # not mean there are any new messages available for the end user to process
+      def idle
+        consumer.on_idle
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -114,15 +119,17 @@ module Karafka
       # @return [Object] cached consumer instance
       def consumer
         @consumer ||= begin
-          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(@topic)
+          topic = @coordinator.topic
+
+          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
 
-          consumer = @topic.consumer_class.new
+          consumer = topic.consumer_class.new
           # We use singleton class as the same consumer class may be used to process different
           # topics with different settings
           consumer.singleton_class.include(strategy)
-          consumer.topic = @topic
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
+          consumer.coordinator = @coordinator
 
           consumer
         end